rknn教程
RKNN
- 安装
-
- api
-
- rknn.config
- rknn.build
- rknn.load_rknn
- rknn.init_runtime
- rknn.inference
- inference
-
- 基于onnx三个输出的推理
- 基于onnx一个输出的推理
安装
现在github下载
api
rknn.config
def config(self,
batch_size=100,
caffe_mean_file=None,
dtype='float32',
epochs=-1,
force_gray=None,
input_fitting='scale',
input_normalization=None,
mean_file=None,
model_data_format='zone',
model_quantize=None,
optimize='Default',
quantized_dtype='asymmetric_affine-u8',
quantized_moving_alpha=0.01,
quantized_algorithm='normal',
quantized_divergence_nbins=1024,
mmse_epoch=3,
random_brightness=None,
random_contrast=None,
random_crop=None,
random_flip=None,
random_mirror=None,
reorder_channel=None,
restart=False,
samples=-1,
need_horizontal_merge=False,
deconv_merge=True,
conv_mul_merge=True,
quantized_hybrid=False,
output_optimize=0,
remove_tensorflow_output_permute=False,
optimization_level=3,
target_platform=None,
mean_values=None,
std_values=None,
channel_mean_value=None,
force_builtin_perm=False,
do_sparse_network=True):
rknn.build
rknn.build(do_quantization,dataset,pre_compile,rknn_batch_size)
pre_compile：预编译开关，如果设置成 True，可以减小模型大小，及模型在硬件设备
上的首次启动速度。但是打开这个开关后，构建出来的模型就只能在硬件平台上运
行，无法通过模拟器进行推理或性能评估。如果硬件有更新，则对应的模型要重新构
建。
rknn.load_rknn
rknn.load_rknn(path,load_model_in_npu)
是否直接加载 npu 中的 rknn 模型。其中 path 为 rknn 模型在 npu
35
中的路径。只有当 RKNN-Toolkit 运行在 RK3399Pro Linux 开发板或连有 NPU 设备
的 PC 上时才可以设为 True。默认值为 False。
rknn.init_runtime
rknn.init_runtime(target,device_id,perf_debug,eval_mem,async_mode)
target：目标硬件平台，目前支持“rk3399pro”、“rk1806”、“rk1808”、“rv1109”、
“rv1126”。默认为 None，即在 PC 使用工具时，模型在模拟器上运行，在 RK3399Pro
Linux 开发板运行时，模型在 RK3399Pro 自带 NPU 上运行，否则在设定的 target 上
运行。其中“rk1808”包含了 TB-RK1808 AI 计算棒
perf_debug：进行性能评估时是否开启 debug 模式。在 debug 模式下，可以获取到每
一层的运行时间，否则只能获取模型运行的总时间。默认值为 False。
eval_mem: 是否进入内存评估模式。进入内存评估模式后，可以调用 eval_memory 接
口获取模型运行时的内存使用情况。默认值为 False。
async_mode：是否使用异步模式。调用推理接口时，涉及设置输入图片、模型推理、
36
获取推理结果三个阶段。如果开启了异步模式，设置当前帧的输入将与推理上一帧同
时进行，所以除第一帧外，之后的每一帧都可以隐藏设置输入的时间，从而提升性能。
在异步模式下，每次返回的推理结果都是上一帧的。该参数的默认值为 False。
rknn.inference
rknn.inference(inputs,data_type,data_format,inputs_pass_through)
inputs：待推理的输入，如经过 cv2 处理的图片。格式是 ndarray list。
data_type：输入数据的类型，可填以下值： ’float32’, ‘float16’, ‘int8’, ‘uint8’, ‘int16’。默认值为’uint8’。
data_format：数据模式，可以填以下值: “nchw”, “nhwc”。默认值为’nhwc’。这两个的不同之处在于 channel 放置的位置。
inputs_pass_through: 将输入透传给 NPU 驱动。非透传模式下，在将输入传给 NPU 驱动之前，工具会对输入进行减均值、除方差等操作；而透传模式下，不会做这些操作。
这个参数的值是一个数组，比如要透传 input0，不透彻 input1，则这个参数的值为[1,0]。默认值为 None，即对所有输入都不透传。
inference
基于onnx三个输出的推理
import os
import numpy as np
import cv2
from rknn.api import RKNN
ONNX_MODEL = 'best.onnx'
RKNN_MODEL = 'yolov.rknn'
IMG_PATH = '3.jpg'
DATASET = 'lh.txt'
NMS_THRESH=0.5
IMG_SIZE=640
strides=[8,16,32]
anchors=np.array([10,13, 16,30, 33,23,30,61, 62,45, 59,119,116,90, 156,198, 373,326])
ahchors=anchors.reshape((3,1,3,1,1,2))
def nms_boxes(boxes, scores):
x = boxes[:, 0]
y = boxes[:, 1]
w = boxes[:, 2] - boxes[:, 0]
h = boxes[:, 3] - boxes[:, 1]
areas = w * h
order = scores.argsort()[::-1]
keep = []
while order.size > 0:
i = order[0]
keep.append(i)
xx1 = np.maximum(x[i], x[order[1:]])
yy1 = np.maximum(y[i], y[order[1:]])
xx2 = np.minimum(x[i] + w[i], x[order[1:]] + w[order[1:]])
yy2 = np.minimum(y[i] + h[i], y[order[1:]] + h[order[1:]])
w1 = np.maximum(0.0, xx2 - xx1 + 0.00001)
h1 = np.maximum(0.0, yy2 - yy1 + 0.00001)
inter = w1 * h1
ovr = inter / (areas[i] + areas[order[1:]] - inter)
inds = np.where(ovr <= NMS_THRESH)[0]
order = order[inds + 1]
keep = np.array(keep)
return keep
def sigmoid(x):
s = 1 / (1 + np.exp(-x))
return s
def make_grid(nx=20, ny=20):
X, Y = np.meshgrid(np.arange(0, nx, 1),np.arange(0, ny, 1))
return np.stack((X, Y), 2).reshape((1, 1, ny, nx, 2))
def my_process(outputs,i):
b,_,ny,nx=outputs.shape
output = np.transpose(outputs.reshape(1, 3, 7, ny, nx),(0, 1, 3, 4, 2))
output=np.ascontiguousarray(output)
y = sigmoid(output)
grid = make_grid(nx, ny)
y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + grid) * strides[i] # xy
y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * anchors[i]
y=y.reshape(-1,7)
return y
def letterbox(im, new_shape=(640, 640), color=(0, 0, 0)):
# Resize and pad image while meeting stride-multiple constraints
shape = im.shape[:2] # current shape [height, width]
if isinstance(new_shape, int):
new_shape = (new_shape, new_shape)
# Scale ratio (new / old)
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
# Compute padding
ratio = r, r # width, height ratios
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding
dw /= 2 # divide padding into 2 sides
dh /= 2
if shape[::-1] != new_unpad: # resize
im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border
return im, ratio, (dw, dh)
if __name__ == '__main__':
# Create RKNN object
rknn = RKNN()
if not os.path.exists(ONNX_MODEL):
print('model not exist')
exit(-1)
# pre-process config
print('--> Config model')
rknn.config(reorder_channel='0 1 2',
mean_values=[[0, 0, 0]],
std_values=[[255, 255, 255]],
optimization_level=3)
print('done')
# Load ONNX model
print('--> Loading model')
ret = rknn.load_onnx(model=ONNX_MODEL)
if ret != 0:
print('Load yolov5 failed!')
exit(ret)
print('done')
# Build model
print('--> Building model')
ret = rknn.build(do_quantization=False, dataset=DATASET)
if ret != 0:
print('Build yolov5 failed!')
exit(ret)
print('done')
# Export RKNN model
print('--> Export RKNN model')
ret = rknn.export_rknn(RKNN_MODEL)
if ret != 0:
print('Export yolov5rknn failed!')
exit(ret)
print('done')
# init runtime environment
print('--> Init runtime environment')
# ret = rknn.init_runtime()
ret = rknn.init_runtime(device_id="0")
if ret != 0:
print('Init runtime environment failed')
exit(ret)
print('done')
# Set inputs
img = cv2.imread(IMG_PATH)
img, ratio, (dw, dh) = letterbox(img, new_shape=(IMG_SIZE, IMG_SIZE))
img = cv2.resize(img, (640, 640), interpolation=cv2.INTER_LINEAR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
color=[127,210,155]
# Inference
print('--> Running model')
outputs = rknn.inference(inputs=[img])
results=[]
for i in range(3):
y=my_process(outputs[i],i)
results.append(y)
result=np.concatenate((results[0],results[1]),axis=0)
result=np.concatenate((result,results[2]),axis=0)
for i in range(result.shape[0]):
if result[i][4]>0.3:
xmin = int(result[i][0]-result[i][2]/2)
ymin = int(result[i][1]-result[i][3]/2)
xmax = int(result[i][0]+result[i][2]/2)
ymax = int(result[i][1]+result[i][3]/2)
print(xmin,ymin,xmax,ymax)
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), color, thickness=3, lineType=cv2.LINE_AA)
cv2.imwrite("11.jpg", img)
rknn.release()
要注意的是没有做nms操作，并且resize图片使用的是letterbox，但最后没有按此方式映射坐标，自己写个简单影射即可，即坐标%ratio-dw即可。主要把torch的后处理过程用python实现一遍详见my_process过程。
基于onnx一个输出的推理
import os
import urllib
import traceback
import time
import sys
import numpy as np
import cv2
from rknn.api import RKNN
ONNX_MODEL = 'best1.onnx'
RKNN_MODEL = 'yolov1.rknn'
IMG_PATH = '6.jpg'
DATASET = 'lh.txt'
OBJ_THRESH = 0.5
NMS_THRESH = 0.6
IMG_SIZE = 640
def nms_boxes(boxes, scores):
"""Suppress non-maximal boxes.
# Arguments
boxes: ndarray, boxes of objects.
scores: ndarray, scores of objects.
# Returns
keep: ndarray, index of effective boxes.
"""
x = boxes[:, 0]
y = boxes[:, 1]
w = boxes[:, 2] - boxes[:, 0]
h = boxes[:, 3] - boxes[:, 1]
areas = w * h
order = scores.argsort()[::-1]
keep = []
while order.size > 0:
i = order[0]
keep.append(i)
xx1 = np.maximum(x[i], x[order[1:]])
yy1 = np.maximum(y[i], y[order[1:]])
xx2 = np.minimum(x[i] + w[i], x[order[1:]] + w[order[1:]])
yy2 = np.minimum(y[i] + h[i], y[order[1:]] + h[order[1:]])
w1 = np.maximum(0.0, xx2 - xx1 + 0.00001)
h1 = np.maximum(0.0, yy2 - yy1 + 0.00001)
inter = w1 * h1
ovr = inter / (areas[i] + areas[order[1:]] - inter)
inds = np.where(ovr <= NMS_THRESH)[0]
order = order[inds + 1]
keep = np.array(keep)
return keep
def plt_img(outputs,img):
output=outputs[0,:,:]
color=[127,0,255]
for i in range(output.shape[0]):
if output[i][4]>0.3:
print("ok")
xmin=int(output[i][0]-output[i][2]/2)
ymin = int(output[i][1]-output[i][3]/2)
xmax = int(output[i][0]+output[i][2]/2)
ymax = int(output[i][1]+output[i][3]/2)
print(xmin,ymin,xmax,ymax)
cv2.rectangle(img, (xmin,ymin), (xmax,ymax), color, thickness=3, lineType=cv2.LINE_AA)
cv2.imwrite("1.jpg",img)
def letterbox(im, new_shape=(640, 640), color=(0, 0, 0)):
# Resize and pad image while meeting stride-multiple constraints
shape = im.shape[:2] # current shape [height, width]
if isinstance(new_shape, int):
new_shape = (new_shape, new_shape)
# Scale ratio (new / old)
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
# Compute padding
ratio = r, r # width, height ratios
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding
dw /= 2 # divide padding into 2 sides
dh /= 2
if shape[::-1] != new_unpad: # resize
im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border
return im, ratio, (dw, dh)
if __name__ == '__main__':
# Create RKNN object
rknn = RKNN()
if not os.path.exists(ONNX_MODEL):
print('model not exist')
exit(-1)
# pre-process config
print('--> Config model')
rknn.config(reorder_channel='0 1 2',
mean_values=[[0, 0, 0]],
std_values=[[255, 255, 255]],
optimization_level=3)
print('done')
# Load ONNX model
print('--> Loading model')
ret = rknn.load_onnx(model=ONNX_MODEL)
if ret != 0:
print('Load yolov5 failed!')
exit(ret)
print('done')
# Build model
print('--> Building model')
ret = rknn.build(do_quantization=False, dataset=DATASET)
if ret != 0:
print('Build yolov5 failed!')
exit(ret)
print('done')
# Export RKNN model
print('--> Export RKNN model')
ret = rknn.export_rknn(RKNN_MODEL)
if ret != 0:
print('Export yolov5rknn failed!')
exit(ret)
print('done')
# init runtime environment
print('--> Init runtime environment')
# ret = rknn.init_runtime()
ret = rknn.init_runtime(device_id="0")
if ret != 0:
print('Init runtime environment failed')
exit(ret)
print('done')
# Set inputs
img = cv2.imread(IMG_PATH)
img, ratio, (dw, dh) = letterbox(img, new_shape=(IMG_SIZE, IMG_SIZE))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# Inference
print('--> Running model')
outputs = rknn.inference(inputs=[img])
plt_img(outputs[0], img)
print("Ok")
rknn.release()
只有一个输出的话就没有后处理过程直接将xywh2xyxy即可，不要忘记nms和坐标影射。