Android ncnn

测试

• 下载 NCNN-Android-Vulkan.zip或自行构建 NCNN for Android
• https://github.com/Tencent/ncnn/releases
  
• 将 ncnn-android-vulkan.zip 提取到 app/src/main/jni 中，或者在 app/src/main/jni/CMakeList 中将ncnn_DIR路径更改为您的路径.txt

native方法

• 进行库的加载 static { System.loadLibrary("styletransferncnn");}且包含两个方法

imageview的相关配置

onCreate的button事件绑定

        imageView = (ImageView) findViewById(R.id.imageView);

        Button buttonImage = (Button) findViewById(R.id.buttonImage);
        buttonImage.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View arg0) {
                Intent i = new Intent(Intent.ACTION_PICK);
                i.setType("image/*");
                startActivityForResult(i, SELECT_IMAGE);
            }
        });

startActivityForResult将调用回调函数onActivityResult

• 有关onActivityResult()的用法

• 然后onActivityResult 会调用最后一个函数private Bitmap decodeUri(Uri selectedImage) throws FileNotFoundException

匿名内部类实现的按钮事件绑定

    <Button
        android:id="@+id/buttonDetect"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:text="变-cpu" />

       Button buttonDetect = (Button) findViewById(R.id.buttonDetect);
        buttonDetect.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View arg0) {
                if (yourSelectedImage == null)
                    return;

                getWindow().setFlags(WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE, WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE);
                new Thread(new Runnable() {
                    public void run() {
                        final Bitmap styledImage = runStyleTransfer(false);
                        imageView.post(new Runnable() {
                            public void run() {
                                imageView.setImageBitmap(styledImage);
                                getWindow().clearFlags(WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE);
                            }
                        });
                    }
                }).start();
            }
        });

相关调用

MainActivity
创建StyleTransferNcnn 对象	private StyleTransferNcnn styletransferncnn = new StyleTransferNcnn();
	StyleTransferNcnn 类中包含两个native调用方法
onCreate方法中调用了初始化	boolean ret_init = styletransferncnn.Init(getAssets());
runStyleTransfer方法中	styletransferncnn.StyleTransfer(styledImage, style_type, use_gpu);
MainActivity	private Bitmap decodeUri(Uri selectedImage) throws FileNotFoundException

    private Bitmap runStyleTransfer(boolean use_gpu)
    {
        Bitmap styledImage = yourSelectedImage.copy(Bitmap.Config.ARGB_8888, true);
        styletransferncnn.StyleTransfer(styledImage, style_type, use_gpu);
        return styledImage;
    }

• CMakeLists.txt 引入动态Opencv库 和 静态 NCNN库
• Linux AndroidStudio + opencv + ncnn 学习记录

快速验证效果

参考链接

• https://github.com/Tencent/ncnn/wiki/use-ncnn-with-alexnet.zh

• ncnn模型加载的三种方式

• https://zhuanlan.zhihu.com/p/268327784

• https://github.com/Tencent/ncnn/wiki/faq

onnx转换为ncnn

工具位置

• linux：ncnn-20230223-ubuntu-2004-shared\bin
  
• windows :ncnn-20230223-windows-vs2017\x64\bin
  

转换命令

• ./onnx2ncnn my_mobileface-sim.onnx my_mobileface.param my_mobileface.bin

• 或使用默认名称./onnx2ncnn my_mobileface-sim.onnx

• 生成.bin与.param文件

加载

ncnn::Net net;
net.load_param("alexnet.param");
net.load_model("alexnet.bin");

• 例如：
  

ANDROID中可以使用的方法

#if __ANDROID_API__ >= 9
#if NCNN_STRING
    // convenient load network structure from android asset plain param file
    int load_param(AAsset* asset);
    int load_param(AAssetManager* mgr, const char* assetpath);
#endif // NCNN_STRING
    // convenient load network structure from android asset binary param file
    int load_param_bin(AAsset* asset);
    int load_param_bin(AAssetManager* mgr, const char* assetpath);

    // convenient load network weight data from android asset model file
    int load_model(AAsset* asset);
    int load_model(AAssetManager* mgr, const char* assetpath);
#endif // __ANDROID_API__ >= 9

推理

执行前向网络，获得计算结果

#include "net.h"
ncnn::Mat in;// input blob as above
ncnn::Mat out;
ncnn::Extractor ex = net.create_extractor();
ex.set_light_mode(true);
ex.input("data", in);
ex.extract("prob", out);

• 注：获取 Mat 中的输出数据，多线程等详见参链接

安全加载

• 去除可见字符串，上边例子中 param 描述文件是明文的，如果放在 APP 分发出去容易被窥探到网络结构（说得好像不明文就看不到一样 使用 ncnn2mem 工具转换为二进制描述文件和内存模型，生成 alexnet.param.bin 和两个静态数组的代码文件

• ncnn2mem alexnet.param alexnet.bin alexnet.id.h alexnet.mem.h

• 加载二进制的 param.bin 和 bin，没有可见字符串，适合 APP 分发模型资源

ncnn::Net net;
net.load_param_bin("alexnet.param.bin");
net.load_model("alexnet.bin");

• 从内存引用加载网络和模型，没有可见字符串，模型数据全在代码里头，没有任何外部文件 另外，android apk 打包的资源文件读出来也是内存块

#include "alexnet.mem.h"
ncnn::Net net;
net.load_param(alexnet_param_bin);
net.load_model(alexnet_bin);

推理

• 如果是二进制的 param.bin 方式，没有可见字符串，利用 XXX.id.h 的枚举来代替 blob 的名字

#include "net.h"
#include "alexnet.id.h"
ncnn::Mat in;// input blob as above
ncnn::Mat out;
ncnn::Extractor ex = net.create_extractor();
ex.set_light_mode(true);
ex.input(alexnet_param_id::BLOB_data, in);
ex.extract(alexnet_param_id::BLOB_prob, out);

native 函数

#include 
#include 
#include 

#include 

#include 
#include 

// ncnn
#include "net.h"
#include "benchmark.h"

#include "styletransfer.id.h"
#include "styletransfer.param.bin.h"

static ncnn::UnlockedPoolAllocator g_blob_pool_allocator;
static ncnn::PoolAllocator g_workspace_pool_allocator;

static ncnn::Net styletransfernet[5];

extern "C" {

JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "StyleTransferNcnn", "JNI_OnLoad");

    ncnn::create_gpu_instance();

    return JNI_VERSION_1_4;
}

JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "StyleTransferNcnn", "JNI_OnUnload");

    ncnn::destroy_gpu_instance();
}

// public native boolean Init(AssetManager mgr);
JNIEXPORT jboolean JNICALL Java_com_tencent_styletransferncnn_StyleTransferNcnn_Init(JNIEnv* env, jobject thiz, jobject assetManager)
{
    ncnn::Option opt;
    opt.lightmode = true;
    opt.num_threads = 4;
    opt.blob_allocator = &g_blob_pool_allocator;
    opt.workspace_allocator = &g_workspace_pool_allocator;

    // use vulkan compute
    if (ncnn::get_gpu_count() != 0)
        opt.use_vulkan_compute = true;

    AAssetManager* mgr = AAssetManager_fromJava(env, assetManager);

    const char* model_paths[5] = {"candy.bin", "mosaic.bin", "pointilism.bin", "rain_princess.bin", "udnie.bin"};
    for (int i=0; i<5; i++)
    {
        styletransfernet[i].opt = opt;

        int ret0 = styletransfernet[i].load_param(styletransfer_param_bin);
        int ret1 = styletransfernet[i].load_model(mgr, model_paths[i]);

        __android_log_print(ANDROID_LOG_DEBUG, "StyleTransferNcnn", "load %d %d", ret0, ret1);
    }

    return JNI_TRUE;
}

// public native Bitmap StyleTransfer(Bitmap bitmap, int style_type, boolean use_gpu);
JNIEXPORT jboolean JNICALL Java_com_tencent_styletransferncnn_StyleTransferNcnn_StyleTransfer(JNIEnv* env, jobject thiz, jobject bitmap, jint style_type, jboolean use_gpu)
{
    if (style_type < 0 || style_type >= 5)
        return JNI_FALSE;

    if (use_gpu == JNI_TRUE && ncnn::get_gpu_count() == 0)
        return JNI_FALSE;

    double start_time = ncnn::get_current_time();

    AndroidBitmapInfo info;
    AndroidBitmap_getInfo(env, bitmap, &info);
    if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
        return JNI_FALSE;

    int width = info.width;
    int height = info.height;

    const int downscale_ratio = 2;

    // ncnn from bitmap
    ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_RGB, width / downscale_ratio, height / downscale_ratio);

    // styletransfer
    ncnn::Mat out;
    {
        ncnn::Extractor ex = styletransfernet[style_type].create_extractor();

        ex.set_vulkan_compute(use_gpu);

        ex.input(styletransfer_param_id::BLOB_input1, in);

        ex.extract(styletransfer_param_id::BLOB_output1, out);
    }

    // ncnn to bitmap
    out.to_android_bitmap(env, bitmap, ncnn::Mat::PIXEL_RGB);

    double elasped = ncnn::get_current_time() - start_time;
    __android_log_print(ANDROID_LOG_DEBUG, "StyleTransferNcnn", "%.2fms   styletransfer", elasped);

    return JNI_TRUE;
}

}

construct input from android Bitmap

error

ndk 版本问题

• https://github.com/Tencent/ncnn/issues/2573
• No toolchains found in the NDK toolchains folder for ABI with prefix: mips64el-linux-android

CG

• github https://github.com/tencent/ncnn

project

• https://github.com/Arctanxy/DeepLearningDeployment
• https://github.com/nihui/ncnn-android-styletransfer