pcDuino9 AI视觉边缘计算开发套件开发使用分享
pcDuino9 AI视觉边缘计算开发套件是一款针对图像分析的开发套件，套件包含采用hi3516D方案的普通IPC和pcDuino9嵌入式AI视觉边缘计算模块。套件主要是通过pcDuino9嵌入式AI视觉边缘计算模块，来处理从hi3516D上获取的视频流，并对视频流进行一系列的分析处理。该套件为开发者搭建了基础开发环境，提供了由pcDuino9从hi3516D上获取的视频帧，开发者可在帧图上自由的实现各种图片开发，例如人脸检测、车牌检测、物体检测等AI图像分析处理功能。同时提供开放的API接口，开发者可以把在pcDuino9上处理完成的信息回传到hi3516D上，并可在hi3516D上对检测到的图片进行删选，然后通过FTP输出。
基本框架
已提供环境：
[if !supportLists]1. [endif]pcDuino9从hi3516D获取视频帧
[if !supportLists]2. [endif]把开发者分析处理后的信息回传到hi3516D上
[if !supportLists]3. [endif]在hi3516D上对分析处理的图片进行质量筛选，选择质量优的图片通过FTP输出
开发举例：
[if !supportLists]1. [endif]在pcDuino9上对帧图进行各种开发，如：人脸检测、车牌检测、物体检测等AI图像分析处理。
[if !supportLists]2. [endif]开发者可对通过FTP输出的分析检测的图片进行其他应用上的使用
[if !supportLists]3. [endif]………
开放SDK资料
[if !supportLists]1. [endif]如何获取hi3516D视频帧？
HI3516将BT1120数据传输到PCDUINO
9。PCDUINO 9通过LT8918将BT1120数据转换成MIPI信号，然后接收。目前支持640x480 60fps帧数据传输。我们封装了协议层，使其类似于opencv样式。在读取帧数据时，我们将返回帧号，该帧号由3516传输，是用于数据交互的同步帧号。
#include
#include
#include "C4L2.hpp"
#include
using namespace std;
using namespace cv;
int main(int argc,char *argv[])
{
string dev = "/dev/video2";
C4L2Capture *cap = new C4L2Capture;
int ret = cap->initialzer(dev);
if(ret < 0)
{
cout <<"C4L2Capture
initialze failed... \n";
return -1;
}
Mat frame;
while(true)
{
unsigned int syn = 0;
cap->read(frame, syn);
if(frame.empty())
{
cout << "this frame is empty
...\n";
break;
}
cv::imshow("DeepCam LLC",frame);
cv::waitKey(10);
}
cap->destroy();
delete cap;
return 0;
}
[if !supportLists]2. [endif]如何优化网络并运行物体检测？
PcDuino9有一个四核Cortex-A17 1.8GHz的CPU和一个ARM MALI-T764的GPU。如果您想在CPU上运行网络，可以参考NCNN；如果您想在GPU上运行网络，可以参考MACE。
CPU Benchmark:
物体检测：
#include
#include
#include
#include
#include
#include "net.h"
class Noop : public ncnn::Layer {};
DEFINE_LAYER_CREATOR(Noop)
struct Object
{
cv::Rect_
int label;
float prob;
};
static int detect_mobilenetv2(const cv::Mat& bgr, std::vector
{
ncnn::Net mobilenetv2;
mobilenetv2.register_custom_layer("Silence", Noop_layer_creator);
// original pretrained model fromhttps://github.com/chuanqi305/MobileNetv2-SSDLite
// https://github.com/chuanqi305/MobileNetv2-
SSDLite/blob/master/ssdlite/voc/deploy.prototxt
mobilenetv2.load_param("mobilenetv2_ssdlite_voc.param");
mobilenetv2.load_model("mobilenetv2_ssdlite_voc.bin");
const int target_size = 300;
int img_w = bgr.cols;
int img_h = bgr.rows;
ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR,
bgr.cols, bgr.rows, target_size, target_size);
const float mean_vals[3] = {127.5f, 127.5f, 127.5f};
const float norm_vals[3] = {1.0/127.5,1.0/127.5,1.0/127.5};
in.substract_mean_normalize(mean_vals, norm_vals);
ncnn::Extractor ex = mobilenetv2.create_extractor();
ex.set_light_mode(true);
ex.set_num_threads(4);
ex.input("data", in);
ncnn::Mat out;
ex.extract("detection_out",out);
// printf("%d %d %d\n", out.w, out.h, out.c);
objects.clear();
for (int i=0; i { const float* values = out.row(i); Object object; object.label = values[0]; object.prob = values[1]; object.rect.x = values[2] * img_w; object.rect.y = values[3] * img_h; object.rect.width = values[4] * img_w - object.rect.x; object.rect.height = values[5] * img_h - object.rect.y; objects.push_back(object); } return 0; } static void draw_objects(const cv::Mat& bgr, const std::vector { static const char* class_names[] = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; cv::Mat image = bgr.clone(); for (size_t i = 0; i < objects.size(); i++) { const Object& obj = objects[i]; fprintf(stderr, "%d = %.5f at %.2f %.2f %.2f x %.2f\n", obj.label, obj.prob, obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height); cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); char text[256]; sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); int baseLine = 0; cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); int x = obj.rect.x; int y = obj.rect.y - label_size.height - baseLine; if (y < 0) y = 0; if (x + label_size.width > image.cols) x = image.cols - label_size.width; cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), cv::Scalar(255, 255, 255), CV_FILLED); cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); } cv::imshow("image", image); cv::waitKey(0); } int main(int argc, char** argv) { if (argc != 2) { fprintf(stderr, "Usage: %s [imagepath]\n", argv[0]); return -1; } const char* imagepath = argv[1]; cv::Mat m = cv::imread(imagepath, CV_LOAD_IMAGE_COLOR); if (m.empty()) { fprintf(stderr, "cv::imread %s failed\n", imagepath); return -1; } std::vector detect_mobilenetv2(m, objects); draw_objects(m, objects); return 0; } [if !supportLists]3. [endif]如何和IPC通信？ 我们提供一个开放式API接口。开发人员可以将pcDuino9上处理的信息返回到HI3516D，在HI3516D上对检测到的图像进行优化选择，然后通过ftp输出。 #include #include "json/json.h" #include "spi_slave.hpp" using namespace std; struct DetectInfo { int ID; //tracker ID int x; // bbox left top x int y; // bbox left top y int w; // bbox width int h; // bbox height; int syn; // sync frame count int quality; // object quality (0 -- 100) int confidence; // object confidence (0 -- 100) }; void detect_out(std::vector { Json::Value root; Json::Value faces; Json::FastWriter writer; for(int i = 0; i < detectBoxs.size(); i++) { Json::Value face; face["x"] = detectBoxs[i].x; face["y"] = detectBoxs[i].y; face["w"] = detectBoxs[i].w; face["h"] = detectBoxs[i].h; face["id"] = detectBoxs[i].ID; face["s"] = detectBoxs[i].ID; face["q"] = detectBoxs[i].quality; face["c"] = detectBoxs[i].confidence; faces.append(face); } root["faces"] = faces; message = writer.write(root); } int main(int argc, char *argv[]) { unsigned char SOF; spi_slave *rk3288_spi = new spi_slave; if(rk3288_spi->init() < 0) { printf("spi test spi_init error.\n"); return -1; } unsigned char *r_buf = new unsigned char[MAX_DATA_LENGTH]; unsigned char *w_buf = new unsigned char[MAX_DATA_LENGTH]; while (true) { std::string message; std::vector //Do object detect get message detect_out(detectBoxs, message); SOF = 0; memset(r_buf, 0, MAX_DATA_LENGTH); strcpy((char *)w_buf,message.c_str()); int ret = rk3288_spi->spi_slave_xfer(r_buf,MAX_DATA_LENGTH, w_buf, MAX_DATA_LENGTH, &SOF); if(ret > 0 ) { memset(w_buf, 0, MAX_DATA_LENGTH); if(SPI_M_TX == SOF) { std::cout <<"recv : " < usleep(30000); } } else { std::cout<<"get message failed!\n"; break; } } return 0; } 硬件介绍 pcDuino9嵌入式AI视觉边缘计算模块介绍 接口图 引脚图 原文链接： http://linksprite.com/wiki/index.php?title=PcDuino9_AI_visual_edge_computing_development_kit D