安卓相机实时帧处理 Day02 《OpenCV Camera搭建》

border="0" width="330" height="86" src="//music.163.com/outchain/player?type=2&id=29807017&auto=1&height=66">

前言

既然环境搭好了，开始玩相机（作死的开始。。。）
在前面环境搭建的基础上继续

实现方案：
采用OpenCV Camera框架 + JNI实现；
工具：
Android Studio 3.3.2 、NDK R16、OpenCV for Android 3.4.1

第一步：修改UI文件

Step 1 ：修改 activity_main.xml

<RelativeLayout
    xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".MainActivity">


    <Button
        android:id="@+id/bt_toCanny"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentStart="true"
        android:layout_alignParentTop="true"
        android:layout_marginStart="11dp"
        android:layout_marginTop="29dp"
        android:text="Canny"/>

    <Button
        android:id="@+id/bt_camera"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentTop="true"
        android:layout_alignParentEnd="true"
        android:layout_marginTop="30dp"
        android:layout_marginEnd="11dp"
        android:text="相机"/>

    <ImageView
        android:id="@+id/imageView"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_centerHorizontal="true"
        android:layout_centerVertical="true"
        android:contentDescription="图片预览"
        android:scaleType="fitCenter"
        android:src="@drawable/test"/>

RelativeLayout>

Step 1 ：添加相机预览UI activity_cameraxml

<RelativeLayout
    xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".CameraView">

    <RadioGroup
        android:id="@+id/radioGroup"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentStart="true"
        android:layout_alignParentLeft="true"
        android:layout_alignParentTop="true"
        android:orientation="horizontal"
        tools:ignore="RtlHardcoded">

        <RadioButton
            android:id="@+id/backCameraOption"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:text="@string/sw_camera"/>
    RadioGroup>

    <org.opencv.android.JavaCameraView
        android:id="@+id/cv_camera"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:layout_below="@+id/radioGroup"
        android:visibility="gone"/>

    <Switch
        android:id="@+id/sw_color"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentTop="true"
        android:layout_alignParentEnd="true"
        android:layout_marginTop="0dp"
        android:layout_marginEnd="3dp"
        android:text="@string/switch_color"
        tools:ignore="RelativeOverlap"/>

RelativeLayout>

第二步：修改app/AndroidManifest.java

<manifest xmlns:android="http://schemas.android.com/apk/res/android"
          xmlns:tools="http://schemas.android.com/tools"
          package="cn.angry.opencvcamera">

    <uses-permission android:name="android.permission.CAMERA"/>
    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>

    <uses-feature android:name="android.hardware.Camera"/>
    <uses-feature android:name="android.hardware.Camera.flash"/>
    <uses-feature android:name="android.hardware.Camera.autofocus"/>

    <application
        android:allowBackup="true"
        android:icon="@mipmap/ic_launcher"
        android:label="@string/app_name"
        android:roundIcon="@mipmap/ic_launcher_round"
        android:supportsRtl="true"
        android:theme="@style/AppTheme"
        tools:ignore="GoogleAppIndexingWarning">
        
        <activity
            android:name=".CameraView"
            android:label="@string/title_activity_camera_view"
            android:theme="@style/AppTheme.NoActionBar">
        activity>

        <activity
            android:name=".MainActivity"
            android:screenOrientation="portrait">
            <intent-filter>
                <action android:name="android.intent.action.MAIN"/>
                <category android:name="android.intent.category.LAUNCHER"/>
            intent-filter>
        activity>

        <meta-data
            android:name="com.google.android.gms.ads.APPLICATION_ID"
            android:value="ca-app-pub-4910454923329631~7109328246"/>

    application>
manifest>

第三步：修改MAinActivity.java

	public class MainActivity extends AppCompatActivity {
    private static int cannyID = 0;
    private static final String TAG = "OpenCV";
    private InterstitialAd mInterstitialAd;

    // OpenCV库静态加载并初始化
    static {
        if (OpenCVLoader.initDebug()) {
            Log.i(TAG, "Load successfully...");
        } else {
            Log.i(TAG, "Fail to load...");
        }
    }

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        Button btCanny = findViewById(R.id.bt_toCanny);
        btCanny.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                cannyID = (cannyID + 1) % 2;
                imageToCanny();
            }
        });

        Button btCamera = findViewById(R.id.bt_camera);
        btCamera.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                Intent intent = new Intent(MainActivity.this.getApplicationContext(), CameraView.class);
                startActivity(intent);
            }
        });

    }

    private void imageToCanny() {
        ImageView imageView = findViewById(R.id.imageView);
        Bitmap image = BitmapFactory.decodeResource(this.getResources(), R.drawable.test);
        if (cannyID == 1) {
            new JniInterface().getEdge(image);
        }
        imageView.setImageBitmap(image);
    }
}

第三步：添加CameraView.java类

package cn.angry.opencvcamera;

import android.Manifest;
import android.os.Build;
import android.os.Bundle;
import android.support.v4.app.ActivityCompat;
import android.support.v7.app.AppCompatActivity;
import android.view.SurfaceView;
import android.view.View;
import android.view.WindowManager;
import android.widget.RadioButton;
import android.widget.Switch;

import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.core.CvType;
import org.opencv.core.Mat;

/**
 * @author hirah
 */
public class CameraView extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2 {
    private JavaCameraView mCameraView;
    private static int requestPermissionId = 1;
    private static int buildVersion = 23;
    private static int cameraId = 0;
    private static int process = 0;
    private static int colorToGray = 0;
    Mat mFrame;
    Mat mGray;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_camera);

        // 动态权限
        if (Build.VERSION.SDK_INT >= buildVersion) {
            ActivityCompat.requestPermissions(this,
                    new String[]{Manifest.permission.CAMERA, Manifest.permission.WRITE_EXTERNAL_STORAGE},
                    requestPermissionId);
        }

        // 设置窗口
        getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN);
        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

        // 初始化相机
        mCameraView = findViewById(R.id.cv_camera);
        mCameraView.setVisibility(SurfaceView.VISIBLE);
        mCameraView.setCvCameraViewListener(this);

        // 开始预览
        mCameraView.setCameraIndex(0);
        mCameraView.enableView();
        mCameraView.enableFpsMeter();

        // 切换摄像头方法
        RadioButton backOption = findViewById(R.id.backCameraOption);
        backOption.setChecked(true);
        backOption.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                cameraId = (cameraId + 1) % 2;
                cameraSwitch(cameraId);
            }
        });

        Switch btHistogram = findViewById(R.id.sw_histogram);
        btHistogram.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                process = (process + 1) % 2;
            }
        });

        Switch btGray = findViewById(R.id.sw_color);
        btGray.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                colorToGray = (colorToGray + 1) % 2;
            }
        });
    }

    @Override
    public void onCameraViewStarted(int width, int height) {
        mFrame = new Mat(height, width, CvType.CV_8UC4);
        mGray = new Mat(height, width, CvType.CV_8UC1);
    }

    @Override
    public void onCameraViewStopped() {
        mFrame.release();
        mGray.release();
    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
        mFrame = inputFrame.rgba();
        mFrame = process(mFrame);
        return mFrame;
    }

    /*
     ** Frame process method
     */

    private Mat process(Mat frame) {

        if (colorToGray == 1) {
            new JniInterface().colorToGray(frame.getNativeObjAddr(), mGray.getNativeObjAddr());
            return mGray;
        }

        if (process == 1) {
            new JniInterface().histogram(frame.getNativeObjAddr(), mGray.getNativeObjAddr());
            return mGray;
        }

        return frame;
    }

    private void cameraSwitch(int id) {
        cameraId = id;
        mCameraView.setCameraIndex(cameraId);
        if (mCameraView != null) {
            mCameraView.disableView();
        }
        mCameraView.enableView();
        mCameraView.enableFpsMeter();
    }

    @Override
    protected void onPause() {
        super.onPause();
        if (mCameraView != null) {
            mCameraView.disableView();
        }
    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        if (mCameraView != null) {
            mCameraView.disableView();
        }
    }

    @Override
    protected void onResume() {
        super.onResume();
        if (mCameraView != null) {
            mCameraView.setCameraIndex(cameraId);
            mCameraView.enableView();
            mCameraView.enableFpsMeter();
        }
    }
}

第四步：修改OpenCV for Android源码

重点啊，超级重要啊，否者会有意想不到的惊喜

• 当修改旋转后，应修改

 // 在 CameraBridgeViewBase.java 中的 deliverAndDrawFrame方法中添加, 否者切换摄像头可能出差错
 if (mCacheBitmap != null) {
                     mCacheBitmap.recycle();
                     mCacheBitmap = Bitmap.createBitmap(modified.width(), modified.height(), Bitmap.Config.ARGB_8888);
                 }

• 修改全屏

 // 修改 JavaCameraView.java， 否者可能无法全屏预览
 if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                         mScale = Math.min(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);
                     
 
 // update -------------------->
 if ((getLayoutParams().width == LayoutParams.MATCH_PARENT) && (getLayoutParams().height == LayoutParams.MATCH_PARENT))
                         mScale = Math.max(((float)height)/mFrameHeight, ((float)width)/mFrameWidth);

• 修改摄像头预览方向，否者可能会导致预览图相倒置。。。一系列迷

// CameraBridgeViewBase.java 的 deliverAndDrawFrame 中添加
if (bmpValid && mCacheBitmap != null) {
            Canvas canvas = getHolder().lockCanvas();
            if (canvas != null) {

                // added for rotation by hirah
                if (getResources().getConfiguration().orientation == Configuration.ORIENTATION_PORTRAIT && mCameraIndex == 0) {
                    canvas.save();
                    canvas.rotate(180,  (canvas.getWidth()/ 2),(canvas.getHeight()/ 2));
                } else if (getResources().getConfiguration().orientation == Configuration.ORIENTATION_PORTRAIT && mCameraIndex == 1) {
                    canvas.save();
                    canvas.rotate(0,  (canvas.getWidth()/ 2),(canvas.getHeight()/ 2));
                } else {
                    canvas.save();
                }

ennnn，到此结，不对，还有JNI（呸。。。）

第五步：修改native-lib.cpp

#include 
#include 
#include 

using namespace std;
using namespace cv;

extern "C" {

int toGray(Mat img, Mat &gray) {
    cvtColor(img, gray, CV_RGBA2GRAY);
    if (gray.rows == img.rows && gray.cols == img.cols) {
        return 1;
    } else {
        return 0;
    }
}

/* 直方图均衡化算法 */
// 预留算法接口，正在测试新的直方图均衡化算法。。。。。
int Hist_equalization(Mat src, Mat &dst){

}

// Color to gray
JNIEXPORT jint JNICALL
Java_cn_angry_opencvcamera_JniInterface_colorToGray(JNIEnv *env, jobject, jlong addrSrc,
                                                    jlong addrDst) {
    Mat &src = *(Mat *) addrSrc;
    Mat &dst = *(Mat *) addrDst;

    int conv;
    jint retVal;
    conv = toGray(src, dst);
    retVal = (jint) conv;
    return retVal;
}

// Histogram
JNIEXPORT jint JNICALL
Java_cn_angry_opencvcamera_JniInterface_histogram(JNIEnv *env, jobject, jlong addrSrc,
                                                  jlong addrDst) {
    Mat &src = *(Mat *) addrSrc;
    Mat &dst = *(Mat *) addrDst;

    int conv;
    jint retVal;
    conv = Hist_equalization(src, dst);
    retVal = (jint) conv;
    return retVal;
}


// Canny edge detect
JNIEXPORT void JNICALL
Java_cn_angry_opencvcamera_JniInterface_getEdge(JNIEnv *env, jobject, jobject bitmap) {
    AndroidBitmapInfo info;
    void *pixels;

    CV_Assert(AndroidBitmap_getInfo(env, bitmap, &info) >= 0);
    CV_Assert(info.format == ANDROID_BITMAP_FORMAT_RGBA_8888 ||
              info.format == ANDROID_BITMAP_FORMAT_RGB_565);
    CV_Assert(AndroidBitmap_lockPixels(env, bitmap, &pixels) >= 0);
    CV_Assert(pixels);
    if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888) {
        Mat temp(info.height, info.width, CV_8UC4, pixels);
        Mat gray;
        cvtColor(temp, gray, COLOR_RGBA2GRAY);
        Canny(gray, gray, 100, 185);
        cvtColor(gray, temp, COLOR_GRAY2RGBA);
    } else {
        Mat temp(info.height, info.width, CV_8UC2, pixels);
        Mat gray;
        cvtColor(temp, gray, COLOR_RGB2GRAY);
        Canny(gray, gray, 100, 185);
        cvtColor(gray, temp, COLOR_GRAY2RGB);
    }
    AndroidBitmap_unlockPixels(env, bitmap);
}
}

第六步：修改JNIInterface.java

/**
 * @author: hirah
 * @date: 19-3-27 上午11:26
 * @note: This is a jni interface for static loading native function.
 */

public class JniInterface {
    static {
        System.loadLibrary("native-lib");
    }

    // 声明本地Jni函数

    /**
     * This is a method for image processing.
     * You can see detail in native-lib.cpp
     * @param:
     *          bitmap for processing
     * @return:
     *          void
     */
    public native void getEdge(Object bitmap);

    /**
     * This is a method for translating color to gray
     * You can see detail in native-lib.cpp
     *
     * @param:
     *          native address of pending frame
     *          native address of result frame
     * @return:
     *          0, not processed
     *          1, processed successfully
     */
    public native int colorToGray(long srcAdd, long dstAdd);

    /**
     * This is a method for histogram equalization
     * You can see detail in native-lib.cpp
     *
     * @param:
     *          native address of pending frame
     *          native address of result frame
     * @return: int
     *          0, not processed
     *          1, processed successfully
     */
    public native int histogram(long srcAdd, long dstAdd);
}

ennnn，到此应该正式结束，测试效果应该有五个方面

• 图片处理，Canny边缘检测
  
• 相机预览

• 实时帧处理
• 相机切换
• 广告显示（ 开源社会要什么。。。广告）