1.Android 腾讯NCNN3分钟实现 摄像头检测 模型加载 人体部位识别 (人工智能)
NCNN
ncnn 是腾讯提供的移动端框架 非常时候在手机玩
ncnn 是一个为手机端极致优化的高性能神经网络前向计算框架。ncnn 从设计之初深刻考虑手机端的部署和使用。无第三方依赖，跨平台，手机端 cpu 的速度快于目前所有已知的开源框架。基于 ncnn，开发者能够将深度学习算法轻松移植到手机端高效执行，开发出人工智能 APP，将 AI 带到你的指尖。ncnn 目前已在腾讯多款应用中使用，如 QQ，Qzone，微信，天天P图等。
功能概述
- 支持卷积神经网络，支持多输入和多分支结构，可计算部分分支
- 无任何第三方库依赖，不依赖 BLAS/NNPACK 等计算框架
- 纯 C++ 实现，跨平台，支持 android ios 等
- ARM NEON 汇编级良心优化，计算速度极快
- 精细的内存管理和数据结构设计，内存占用极低
- 支持多核并行计算加速，ARM big.LITTLE cpu 调度优化
- 支持基于全新低消耗的 vulkan api GPU 加速
- 整体库体积小于 700K，并可轻松精简到小于 300K
- 可扩展的模型设计，支持 8bit 量化和半精度浮点存储，可导入 caffe/pytorch/mxnet/onnx/darknet 模型
- 支持直接内存零拷贝引用加载网络模型
- 可注册自定义层实现并扩展
- 恩，很强就是了，不怕被塞卷 QvQ
demo功能演示
1.人脸识别
2.身份证，银行卡识别
3.语音识别
4.车票识别
5.人体部位识别
DEMO解决了几个问题：
1.替换了模型，检测不出结果的问题
2.模型加载不了的问题或者奔溃
3.模型加载耗时6s。问题修复
4.识别内存溢出问题修复
demo地址：模型加载和自动识别
https://github.com/nihui/ncnn-android-mobilenetssd
3分钟实现demo：
step1
https://github.com/Tencent/ncnn/releases
download ncnn-android-vulkan-lib.zip or build ncnn for android yourself
下载库ncnn-android-vulkan-lib：
step2
extract ncnn-android-vulkan-lib.zip into app/src/main/jni or change the ncnn path to yours in app/src/main/jni/CMakeLists.txt
配置cmake
android ios 预编译库 20200616 622879a
编译版本，默认配置，android-ndk-r21d，cctools-port 895 + ld64-274.2 + ios 10.2 sdk libc++
ncnn-android-lib 是 android 的静态库(armeabi-v7a + arm64-v8a + x86 + x86_64)
ncnn-android-vulkan-lib 是 android 的静态库(armeabi-v7a + arm64-v8a + x86 + x86_64，包含vulkan支持)
ncnn.framework.zip 是 ios 的静态库(armv7 + arm64 + i386 + x86_64，bitcode)
ncnn-vulkan.framework.zip 是 ios 的静态库(arm64 + x86_64，bitcode，包含vulkan支持，MoltenVK-1.1.82.0)
openmp.framework.zip 是 ios ncnn openmp 运行时静态库(armv7 + arm64 + i386 + x86_64，bitcode)
adreno gpu image存储+fp16p/fp16s/fp16pa/fp16sa优化，在qcom855之前的高通芯片上默认启用，包括全部gpu shader
新增darknet转换器，支持yolov4和efficientnetb0-yolov3(by zhiliu6)
新增simplestl，可替代std::string/std::vector，默认不启用(by scarsty)
新增NCNN_LOGE宏，android自动在adb logcat输出信息(by maxint)
运行时生成spirv，大幅减小gpu库体积
新增python绑定链接
新增查询当前可用gpu显存接口
gpu fp16/fp32转换，buffer/image
源码分析：
第一步：加载模型。量化模型
// init param
{
int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param");
if (ret != 0)
{
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed");
return JNI_FALSE;
}
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success");
}
// init bin
{
int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin");
if (ret != 0)
{
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed");
return JNI_FALSE;
}
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success");
}
第二步：初始化java需要的一些参数
// init jni glue
jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj");
objCls = reinterpret_cast(env->NewGlobalRef(localObjCls));
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start");
constructortorId = env->GetMethodID(objCls, "", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V");
xId = env->GetFieldID(objCls, "x", "F");
yId = env->GetFieldID(objCls, "y", "F");
wId = env->GetFieldID(objCls, "w", "F");
hId = env->GetFieldID(objCls, "h", "F");
labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;");
probId = env->GetFieldID(objCls, "prob", "F");
__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end");
第三步：识别模型
}
double start_time = ncnn::get_current_time();
AndroidBitmapInfo info;
AndroidBitmap_getInfo(env, bitmap, &info);
int width = info.width;
int height = info.height;
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
return NULL;
// ncnn from bitmap
ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300);
// mobilenetssd
std::vector
第4步：根据坐标进行车子的绘制
for (int i = 0; i < objects.length; i++)
{
Log.d("peng","objects"+objects[i].toString());
canvas.drawRect(objects[i].x, objects[i].y, objects[i].x + objects[i].w, objects[i].y + objects[i].h, paint);
// draw filled text inside image
{
String text = objects[i].label + " = " + String.format("%.1f", objects[i].prob * 100) + "%";
float text_width = textpaint.measureText(text);
float text_height = - textpaint.ascent() + textpaint.descent();
float x = objects[i].x;
float y = objects[i].y - text_height;
if (y < 0)
y = 0;
if (x + text_width > rgba.getWidth())
x = rgba.getWidth() - text_width;
canvas.drawRect(x, y, x + text_width, y + text_height, textbgpaint);
canvas.drawText(text, x, y - textpaint.ascent(), textpaint);
}
}
imageView.setImageBitmap(rgba);
}
#include#include #include #include #include #include #include // ncnn #include "net.h" #include "benchmark.h" static ncnn::UnlockedPoolAllocator g_blob_pool_allocator; static ncnn::PoolAllocator g_workspace_pool_allocator; static ncnn::Net mobilenetssd; struct Object { float x; float y; float w; float h; int label; float prob; }; extern "C" { // FIXME DeleteGlobalRef is missing for objCls static jclass objCls = NULL; static jmethodID constructortorId; static jfieldID xId; static jfieldID yId; static jfieldID wId; static jfieldID hId; static jfieldID labelId; static jfieldID probId; JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) { __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnLoad"); ncnn::create_gpu_instance(); return JNI_VERSION_1_4; } JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved) { __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnUnload"); ncnn::destroy_gpu_instance(); } // public native boolean Init(AssetManager mgr); JNIEXPORT jboolean JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Init(JNIEnv* env, jobject thiz, jobject assetManager) { ncnn::Option opt; opt.lightmode = true; opt.num_threads = 4; opt.blob_allocator = &g_blob_pool_allocator; opt.workspace_allocator = &g_workspace_pool_allocator; opt.use_packing_layout = true; // use vulkan compute if (ncnn::get_gpu_count() != 0) opt.use_vulkan_compute = true; AAssetManager* mgr = AAssetManager_fromJava(env, assetManager); mobilenetssd.opt = opt; // init param { int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param"); if (ret != 0) { __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed"); return JNI_FALSE; } __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success"); } // init bin { int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin"); if (ret != 0) { __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed"); return JNI_FALSE; } __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success"); } // init jni glue jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj"); objCls = reinterpret_cast (env->NewGlobalRef(localObjCls)); __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start"); constructortorId = env->GetMethodID(objCls, " ", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V"); xId = env->GetFieldID(objCls, "x", "F"); yId = env->GetFieldID(objCls, "y", "F"); wId = env->GetFieldID(objCls, "w", "F"); hId = env->GetFieldID(objCls, "h", "F"); labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;"); probId = env->GetFieldID(objCls, "prob", "F"); __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end"); return JNI_TRUE; } // public native Obj[] Detect(Bitmap bitmap, boolean use_gpu); JNIEXPORT jobjectArray JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Detect(JNIEnv* env, jobject thiz, jobject bitmap, jboolean use_gpu) { if (use_gpu == JNI_TRUE && ncnn::get_gpu_count() == 0) { return NULL; //return env->NewStringUTF("no vulkan capable gpu"); } double start_time = ncnn::get_current_time(); AndroidBitmapInfo info; AndroidBitmap_getInfo(env, bitmap, &info); int width = info.width; int height = info.height; if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) return NULL; // ncnn from bitmap ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300); // mobilenetssd std::vector