yolov2-Tiny转换到ncnn下移植到android端

之前一篇介绍了yolov2-Tiny在darknet下训练，之后转化为caffe下，最终转换到ncnn下：https://blog.csdn.net/qq_29377279/article/details/83548180
这一篇将记录一下我和师兄继续踩坑android，对于不会安卓，C/C++一般的我们，居然还要结合安卓和C/C++。

先安装好Android Studio，再安装好NDK参考：
https://blog.csdn.net/qq_36982160/article/details/79931741#commentBox
装完之后检查一下：

ndk-build -v

安装完成就开始吧：

先编译好ncnn sdk

我们需要用ndk-build将ncnn打包，这样我们才能在android ndk的代码中使用include

#参数说明：
ANDROID_ABI 是架构名字，"armeabi-v7a" 支持绝大部分手机硬件 
ANDROID_ARM_NEON 是否使用 NEON 指令集，设为 ON 支持绝大部分手机硬件 
ANDROID_PLATFORM 指定最低系统版本，"android-14" 就是 android-4.0

mkdir build-android 
cd build-android 
cmake -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake -DANDROID_ABI="armeabi-v7a" -DANDROID_ARM_NEON=ON -DANDROID_PLATFORM=android-14 .. 

make 

make install

install安装完成之后，去看看我们刚刚的文件夹下多了 include 和lib，include的下是.h文件，lib下是编译后的ncnn的静态库libncnn.a,后面将会用到。

这篇博客 https://blog.csdn.net/qq_36982160/article/details/79931741#commentBox 使用的是ndk开发，我们开始也仿照着进行，一直没有成功，一直探索到发现，好像是我们数据进入detect就会出错，没找出错误的原因，就转换了另一种方法用cmake。
参考：https://blog.csdn.net/qq_33200967/article/details/82421089
我们采用模型参数加密的方法，也可以不加密直接使用。
使用之前ncnn下编译出来的ncnn2mem，命令如下：

./ncnn2mem yolov2_tiny_tarmac.param yolov2_tiny_tarmac.bin yolov2_tiny_tarmac.id.h yolov2_tiny_tarmac.mem.h

生成加密之后的文件：
yolov2_tiny_tarmac.param.bin
yolov2_tiny_tarmac.bin
yolov2_tiny_tarmac.id.h
yolov2_tiny_tarmac.mem.h
我们之后将要用到
yolov2_tiny_tarmac.param.bin网络的模型参数
yolov2_tiny_tarmac.bin网络的权重
yolov2_tiny_tarmac.id.h在预测图片的时候使用到

开始Android工程：

在Android Studio建工程时记得选C++11支持。

在app/src/main下创建文件夹assets：

将之前生产的yolov2_tiny_tarmac.param.bin、yolov2_tiny_tarmac.bin复制到该目录；
再创建words.txt，里面为label的名，注意第一个为background背景，我这里只有一个检测目标如下:

在cpp目录下：
复制在使用Ubuntu编译NCNN库部分编译得到的include文件夹，包括里面的C++头文件；
把yolov2_tiny_tarmac.id.h也复制到cpp目录下。
这里看到一个yolov2-jni.cpp，这是jni接口实现，下面将会有实现过程。（对jni可以去看看相关资料，应该可以理解。）

在main目录下创建jniLibs/armeabi-v7a/目录：
把之前使用Ubuntu编译NCNN库部分编译得到的lib文件夹下的libncnn.a复制到该目录。

修改APP目录下的CMakeLists.txt文件：
对照着修改为自己的即可：

# For more information about using CMake with Android Studio, read the
# documentation: https://d.android.com/studio/projects/add-native-code.html

# Sets the minimum version of CMake required to build the native library.

cmake_minimum_required(VERSION 3.4.1)

# Creates and names a library, sets it as either STATIC
# or SHARED, and provides the relative paths to its source code.
# You can define multiple libraries, and CMake builds them for you.
# Gradle automatically packages shared libraries with your APK.

set(ncnn_lib ${CMAKE_SOURCE_DIR}/src/main/jniLibs/armeabi-v7a/libncnn.a)
add_library (ncnn_lib STATIC IMPORTED)
set_target_properties(ncnn_lib PROPERTIES IMPORTED_LOCATION ${ncnn_lib})

add_library( # Sets the name of the library.
        yolov2_jni   

        # Sets the library as a shared library.
        SHARED

        # Provides a relative path to your source file(s).
        src/main/cpp/yolov2-jni.cpp)

# Searches for a specified prebuilt library and stores the path as a
# variable. Because CMake includes system libraries in the search path by
# default, you only need to specify the name of the public NDK library
# you want to add. CMake verifies that the library exists before
# completing its build.

find_library( # Sets the name of the path variable.
        log-lib

        # Specifies the name of the NDK library that
        # you want CMake to locate.
        log)
#find_library( # Sets the name of the path variable.
        #JniGraphics

        # Specifies the name of the NDK library that
        # you want CMake to locate.
        #jnigraphics)

# Specifies libraries CMake should link to your target library. You
# can link multiple libraries, such as libraries you define in this
# build script, prebuilt third-party libraries, or system libraries.

target_link_libraries( # Specifies the target library.
        yolov2_jni
        ncnn_lib
        jnigraphics

        # Links the target library to the log library
        # included in the NDK.
        ${log-lib})

（对应的地方都有说明，我就不写中文注释了。）

修改APP目录下的build.gradle文件：

apply plugin: 'com.android.application'

android {
    compileSdkVersion 28
    defaultConfig {
        applicationId "com.tarmac.yolov2Tiny"
        minSdkVersion 21
        targetSdkVersion 28
        versionCode 1
        versionName "1.0"
        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
        externalNativeBuild {
            cmake {
                cppFlags "-std=c++11 -fopenmp"
                abiFilters "armeabi-v7a"
            }
        }
    }
    buildTypes {
        release {
            minifyEnabled false
            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
        }
    }
    externalNativeBuild {
        cmake {
            path "CMakeLists.txt"
        }
    }
    sourceSets {
        main {
            jniLibs.srcDirs = ["src/main/jniLibs"]
            jni.srcDirs = ['src/cpp']
        }
    }
}

dependencies {
    implementation fileTree(dir: 'libs', include: ['*.jar'])
    implementation 'com.android.support:appcompat-v7:28.0.0'
    implementation 'com.android.support.constraint:constraint-layout:1.1.3'
    testImplementation 'junit:junit:4.12'
    implementation 'com.github.bumptech.glide:glide:4.3.1'
    androidTestImplementation 'com.android.support.test:runner:1.0.2'
    androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2'
}

给main下的AndroidManifest.xml配置文件中添加权限：

好了，所以需要配置的一些东西都搞定了，下面开始写我们需要的测试代码把。

这是我java下的目录：

先修改MainActivity.java中的代码：

package com.tarmac.yolov2Tiny;

import android.Manifest;
import android.app.Activity;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.graphics.Typeface;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.net.Uri;
import android.os.Bundle;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.v4.app.ActivityCompat;
import android.support.v4.content.ContextCompat;
import android.text.method.ScrollingMovementMethod;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.ImageView;
import android.widget.TextView;
import android.widget.Toast;

import com.bumptech.glide.Glide;
import com.bumptech.glide.load.engine.DiskCacheStrategy;
import com.bumptech.glide.request.RequestOptions;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;


public class MainActivity extends Activity {
    private static final String TAG = MainActivity.class.getName();
    private static final int USE_PHOTO = 1001;
    private String camera_image_path;
    private ImageView show_image;
    private TextView result_text;
    private boolean load_result = false;
    private int[] ddims = {1, 3, 416, 416};
    private int model_index = 1;
    private List resultLabel = new ArrayList<>();
    private yolov2Tiny yolov2Tiny = new yolov2Tiny();


    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        try {
            initYolov2Tiny();
        } catch (IOException e) {
            Log.e("MainActivity", "initYolov2Tiny error");
        }
        init_view();
        readCacheLabelFromLocalFile();
    }

    private void initYolov2Tiny() throws IOException {
        byte[] param = null;
        byte[] bin = null;
        {
            InputStream assetsInputStream = getAssets().open("yolov2-tiny_tarmac.param.bin");
            int available = assetsInputStream.available();
            param = new byte[available];
            int byteCode = assetsInputStream.read(param);
            assetsInputStream.close();
        }
        {
            InputStream assetsInputStream = getAssets().open("yolov2-tiny_tarmac1.bin");
            int available = assetsInputStream.available();
            bin = new byte[available];
            int byteCode = assetsInputStream.read(bin);
            assetsInputStream.close();
        }

        load_result = yolov2Tiny.Init(param, bin);
        Log.d("load model", "yolov2tiny_load_model_result:" + load_result);
    }

    // initialize view
    private void init_view() {
        request_permissions();
        show_image = (ImageView) findViewById(R.id.show_image);
        result_text = (TextView) findViewById(R.id.result_text);
        result_text.setMovementMethod(ScrollingMovementMethod.getInstance());
        Button use_photo = (Button) findViewById(R.id.use_photo);
        // use photo click
         use_photo.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View view) {
                if (!load_result) {
                    Toast.makeText(MainActivity.this, "never load model", Toast.LENGTH_SHORT).show();
                    return;
                }
                PhotoUtil.use_photo(MainActivity.this, USE_PHOTO);
            }
        });
    }

    // load label's name
    private void readCacheLabelFromLocalFile() {
        try {
            AssetManager assetManager = getApplicationContext().getAssets();
            BufferedReader reader = new BufferedReader(new InputStreamReader(assetManager.open("words.txt")));
            String readLine = null;
            while ((readLine = reader.readLine()) != null) {
                resultLabel.add(readLine);
            }
            reader.close();
        } catch (Exception e) {
            Log.e("labelCache", "error " + e);
        }
    }

    @Override
    protected void onActivityResult(int requestCode, int resultCode, @Nullable Intent data) {
        String image_path;
        RequestOptions options = new RequestOptions().skipMemoryCache(true).diskCacheStrategy(DiskCacheStrategy.NONE);
        if (resultCode == Activity.RESULT_OK) {
            switch (requestCode) {
                case USE_PHOTO:
                    if (data == null) {
                        Log.w(TAG, "user photo data is null");
                        return;
                    }
                    Uri image_uri = data.getData();

                    //Glide.with(MainActivity.this).load(image_uri).apply(options).into(show_image);

                    // get image path from uri
                    image_path = PhotoUtil.get_path_from_URI(MainActivity.this, image_uri);
                    // predict image
                    predict_image(image_path);
                    break;
            }
        }
    }

    //  predict image
    private void predict_image(String image_path) {
        // picture to float array
        Bitmap bmp = PhotoUtil.getScaleBitmap(image_path);
        Bitmap rgba = bmp.copy(Bitmap.Config.ARGB_8888, true);
        // resize to 416x416
        Bitmap input_bmp = Bitmap.createScaledBitmap(rgba, ddims[2], ddims[3], false);
        try {
            // Data format conversion takes too long
            // Log.d("inputData", Arrays.toString(inputData));
            long start = System.currentTimeMillis();
            // get predict result
            float[] result = yolov2Tiny.Detect(input_bmp);
            long end = System.currentTimeMillis();
            Log.d(TAG, "origin predict result:" + Arrays.toString(result));
            long time = end - start;
            Log.d("result length", "length of result: " + String.valueOf(result.length));
            // show predict result and time
            float[] r = get_max_result(result);

            String show_text = "result：" + Arrays.toString(r) + "\nname：" + resultLabel.get((int) r[0]) + "\nprobability：" + r[1] + "\ntime：" + time + "ms" ;
            result_text.setText(show_text);
            Canvas canvas = new Canvas(rgba);
            //图像上画矩形
            Paint paint = new Paint();
            paint.setColor(Color.RED);
            paint.setStyle(Paint.Style.STROKE);//不填充
            paint.setStrokeWidth(10); //线的宽度
            canvas.drawRect(r[2]*rgba.getWidth(), r[3]*rgba.getHeight(), r[4]*rgba.getWidth(), r[5]*rgba.getHeight(), paint);
            show_image.setImageBitmap(rgba);


        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    // get max probability label
    private float[] get_max_result(float[] result) {
        int num_rs = result.length / 6;
        float maxProp = result[1];
        int maxI = 0;
        for(int i = 1; i permissionList = new ArrayList<>();
        if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
            permissionList.add(Manifest.permission.CAMERA);
        }
        if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) {
            permissionList.add(Manifest.permission.WRITE_EXTERNAL_STORAGE);
        }
        if (ContextCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) {
            permissionList.add(Manifest.permission.READ_EXTERNAL_STORAGE);
        }
        // if list is not empty will request permissions
        if (!permissionList.isEmpty()) {
            ActivityCompat.requestPermissions(this, permissionList.toArray(new String[permissionList.size()]), 1);
        }
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
        switch (requestCode) {
            case 1:
                if (grantResults.length > 0) {
                    for (int i = 0; i < grantResults.length; i++) {
                        int grantResult = grantResults[i]; 
                        if (grantResult == PackageManager.PERMISSION_DENIED) {
                            String s = permissions[i];
                            Toast.makeText(this, s + "permission was denied", Toast.LENGTH_SHORT).show();
                        }
                    } 
                } 
                break;
        }
    }
}

再创建一个PhotoUtil.java图像工具类：

package com.tarmac.yolov2Tiny;

import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.database.Cursor;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.net.Uri;
import android.provider.MediaStore;

public class PhotoUtil {
    // get picture in photo
    public static void use_photo(Activity activity, int requestCode) {
        Intent intent = new Intent(Intent.ACTION_PICK);
        intent.setType("image/*");
        activity.startActivityForResult(intent, requestCode);
    }

    // get photo from Uri
    public static String get_path_from_URI(Context context, Uri uri) {
        String result;
        Cursor cursor = context.getContentResolver().query(uri, null, null, null, null);
        if (cursor == null) {
            result = uri.getPath();
        } else {
            cursor.moveToFirst();
            int idx = cursor.getColumnIndex(MediaStore.Images.ImageColumns.DATA);
            result = cursor.getString(idx);
            cursor.close();
        }
        return result;
    }

    // compress picture
    public static Bitmap getScaleBitmap(String filePath) {
        BitmapFactory.Options opt = new BitmapFactory.Options();
        opt.inJustDecodeBounds = true;
        BitmapFactory.decodeFile(filePath, opt);

        int bmpWidth = opt.outWidth;
        int bmpHeight = opt.outHeight;

        int maxSize = 500;

        // compress picture with inSampleSize
        opt.inSampleSize = 1;
        while (true) {
            if (bmpWidth / opt.inSampleSize < maxSize || bmpHeight / opt.inSampleSize < maxSize) {
                break;
            }
            opt.inSampleSize *= 2;
        }
        opt.inJustDecodeBounds = false;
        return BitmapFactory.decodeFile(filePath, opt);
    }
}

创建一个yolov2Tiny.java类，用于提供JNI接口：

package com.tarmac.yolov2Tiny;
import android.graphics.Bitmap;

public class yolov2Tiny {
    public native boolean Init(byte[] param, byte[] bin);
    public native float[] Detect(Bitmap bitmap);
    static {
        System.loadLibrary("yolov2_jni");
    }
}

在cpp创建yolov2-jni.cpp，就是上面jni接口的C++实现：
主要就是两个函数，一个初始化，加载网络和参数，另一个就是检测预测，对照ncnn中yolov2.cpp实现。

#include 
#include 
#include 
#include 
#include 
// ncnn
#include "include/net.h"
#include "include/opencv.h"
#include "yolov2-tiny_tarmac.id.h"
#include 
#include 
static ncnn::UnlockedPoolAllocator g_blob_pool_allocator;
static ncnn::PoolAllocator g_workspace_pool_allocator;

static ncnn::Mat ncnn_param;
static ncnn::Mat ncnn_bin;
static ncnn::Net ncnn_net;

extern "C" {

// public native boolean Init(byte[] words,byte[] param, byte[] bin);
JNIEXPORT jboolean JNICALL
Java_com_tarmac_yolov2Tiny_yolov2Tiny_Init(JNIEnv *env, jobject obj, jbyteArray param, jbyteArray bin) {
    __android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJni", "enter the jni func");
    // init param
    {
        int len = env->GetArrayLength(param);
        ncnn_param.create(len, (size_t) 1u);
        env->GetByteArrayRegion(param, 0, len, (jbyte *) ncnn_param);
        int ret = ncnn_net.load_param((const unsigned char *) ncnn_param);
        __android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJni", "load_param %d %d", ret, len);
    }

    // init bin
    {
        int len = env->GetArrayLength(bin);
        ncnn_bin.create(len, (size_t) 1u);
        env->GetByteArrayRegion(bin, 0, len, (jbyte *) ncnn_bin);
        int ret = ncnn_net.load_model((const unsigned char *) ncnn_bin);
        __android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJni", "load_model %d %d", ret, len);
    }

    ncnn::Option opt;
    opt.lightmode = true;
    opt.num_threads = 4;//线程数
    opt.blob_allocator = &g_blob_pool_allocator;
    opt.workspace_allocator = &g_workspace_pool_allocator;

    ncnn::set_default_option(opt);

    return JNI_TRUE;
}

// public native String Detect(Bitmap bitmap);
JNIEXPORT jfloatArray JNICALL Java_com_tarmac_yolov2Tiny_yolov2Tiny_Detect(JNIEnv* env, jobject thiz, jobject bitmap)
{
    // ncnn from bitmap
    ncnn::Mat in;
    {
        AndroidBitmapInfo info;
        AndroidBitmap_getInfo(env, bitmap, &info);
//        int origin_w = info.width;
//        int origin_h = info.height;
//        int width = 416;
//        int height = 416;
        int width = info.width;
        int height = info.height;
        if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
            return NULL;

        void* indata;
        AndroidBitmap_lockPixels(env, bitmap, &indata);
        // 把像素转换成data，并指定通道顺序
//        in = ncnn::Mat::from_pixels_resize((const unsigned char*)indata, ncnn::Mat::PIXEL_RGBA2RGB, origin_w, origin_h, width, height);

        in = ncnn::Mat::from_pixels((const unsigned char*)indata, ncnn::Mat::PIXEL_RGBA2RGB, width, height);
        //__android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJniIn", "yolov2_predict_has_input1, in.w: %d; in.h: %d", in.w, in.h);
        AndroidBitmap_unlockPixels(env, bitmap);
    }

    // ncnn_net
    std::vector cls_scores;
    {
        // 减去均值和乘上比例
        const float mean_vals[3] = {0.5f, 0.5f, 0.5f};
        const float scale[3] = {0.007843f, 0.007843f, 0.007843f};
        //__android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJniIn", "yolov2_predict_has_input2, in[0][0]: %f; in[250][250]: %f", in.row(0)[0], in.row(250)[250]);
//        in.substract_mean_normalize(mean_vals, scale);
        in.substract_mean_normalize(mean_vals, scale);
        //__android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJniIn", "yolov2_predict_has_input3, in[0][0]: %f; in[250][250]: %f", in.row(0)[0], in.row(250)[250]);

        ncnn::Extractor ex = ncnn_net.create_extractor();
        // 如果时不加密是使用ex.input("data", in);
//        ex.input(mobilenet_v2_param_id::BLOB_data, in);
        ex.input(yolov2_tiny_tarmac_param_id::BLOB_data, in);
        //ex.input("data",in);
        ncnn::Mat out;
        // 如果时不加密是使用ex.extract("prob", out);
        //ex.extract(mobilenet_v2_param_id::BLOB_prob, out);
        ex.extract(yolov2_tiny_tarmac_param_id::BLOB_detection_out, out);
//        ex.extract("detection-out",out);
//        int output_wsize = in.w;
//        int output_hsize = in.h;
//        jint *in_out[output_wsize * output_hsize * in.c];
//        for(int i=0; iNewIntArray(output_hsize * output_wsize * in.c);
//        env->SetIntArrayRegion(jOutImData, 0, output_hsize * output_wsize * in.c, reinterpret_cast(*in_out));
        //__android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJniOut", "yolov2_predict_has_outcome, out.w: %d; out.h: %d", out.w, out.h);
        //__android_log_print(ANDROID_LOG_DEBUG, "yolov2TinyJniOut", "yolov2_predict_has_outcome_data, out[0][2]: %f;", out.row(0)[2]);
        int output_wsize = out.w;
        int output_hsize = out.h;

        jfloat *output[output_wsize * output_hsize];
        for(int i = 0; i< out.h; i++) {
            for (int j = 0; j < out.w; j++) {
                output[i*output_wsize + j] = &out.row(i)[j];
            }
        }
        jfloatArray jOutputData = env->NewFloatArray(output_wsize);
        if (jOutputData == nullptr) return nullptr;
        env->SetFloatArrayRegion(jOutputData, 0,  output_wsize * output_hsize,
                                 reinterpret_cast(*output));  // copy

        return jOutputData;
    }
}
}

修改启动布局activity_main.xml：
在src/main/res下的layout中：

到此，demo全部完成，编译工程，查看生成的.so：
在build/cmake下

安装运行：

效果还不错。
我在华为平板M5上试了不同的线程数，单线程500ms左右，4线程220ms左右，8线程180ms左右。

我的源码：https://github.com/yuace/yolov2-Tiny-NCNN-Android-demo