【KAWAKO】TVM-使用c++进行推理
目录
前言
修改cpp_deploy.cc文件
修改DeployGraphExecutor()函数
numpy与bin文件的互相转换
numpy转bin
bin转numpy
使用CMakeLists.txt进行编译
运行
前言
在tvm工程的apps目录下，有一个howto_deploy的工程，根据此工程进行修改，可以得到c++推理程序。
修改cpp_deploy.cc文件
DeploySingleOp()函数不需要，直接将其和相关的Verify函数一起删掉。
修改DeployGraphExecutor()函数
读取指定模型，同时获得后面所需的函数
LOG(INFO) << "Running graph executor...";
printf("load in the library\n");
DLDevice dev{kDLCPU, 1};
tvm::runtime::Module mod_factory = tvm::runtime::Module::LoadFromFile("../model_autotune.so");
printf("create the graph executor module\n");
tvm::runtime::Module gmod = mod_factory.GetFunction("default")(dev);printf(" default\n");
tvm::runtime::PackedFunc set_input = gmod.GetFunction("set_input");printf(" set_input\n");
tvm::runtime::PackedFunc get_output = gmod.GetFunction("get_output");printf(" get_output\n");
tvm::runtime::PackedFunc run = gmod.GetFunction("run");printf(" run\n");定义输入输出的变量
printf("Use the C++ API\n");
tvm::runtime::NDArray input = tvm::runtime::NDArray::Empty({1, 1, 640}, DLDataType{kDLFloat, 32, 1}, dev);
tvm::runtime::NDArray input_state = tvm::runtime::NDArray::Empty({1, 2, 128, 2}, DLDataType{kDLFloat, 32, 1}, dev);
tvm::runtime::NDArray output = tvm::runtime::NDArray::Empty({1, 1, 640}, DLDataType{kDLFloat, 32, 1}, dev);
tvm::runtime::NDArray output_state = tvm::runtime::NDArray::Empty({1, 2, 128, 2}, DLDataType{kDLFloat, 32, 1}, dev);从bin文件中读取数据
float input_storage[1 * 1 * 640];
FILE* fp = fopen("../input.bin", "rb");
fread(input->data, 1 * 1 * 640, 4, fp);
fclose(fp);
float input_state_storage[1 * 2 * 128 * 2];
FILE* fp_state = fopen("../input_state.bin", "rb");
fread(input_state->data, 1 * 2 * 128 * 2, 4, fp_state);
fclose(fp_state);将数据输入到网络
printf("set the right input\n");
set_input("input_4", input);
set_input("input_5", input_state);运行推理
struct timeval t0, t1;
int times = 100000; // 3394
gettimeofday(&t0, 0);
printf("run the code\n");
for(int i=0;i得到输出
printf("get the output\n");
get_output(0, output);printf(" 0\n");
get_output(1, output_state);printf(" 1\n");将输出保存到bin文件
FILE* fp_out = fopen("output.bin", "wb");
fwrite(output->data, 1 * 1 * 640, 4, fp_out);
fclose(fp_out);
FILE* fp_out_state = fopen("output_state.bin", "wb");
fwrite(output_state->data, 1 * 2 * 128 * 2, 4, fp_out_state);
fclose(fp_out_state);numpy与bin文件的互相转换
numpy转bin
import numpy as np
import os
input_1 = np.load("./input.npy")
input_2 = np.load("./input_states.npy")
build_dir = "./"
with open(os.path.join(build_dir, "input.bin"), "wb") as fp:
fp.write(input_1.astype(np.float32).tobytes())
with open(os.path.join(build_dir, "input_state.bin"), "wb") as fp:
fp.write(input_2.astype(np.float32).tobytes())bin转numpy
output = np.fromfile("./output.bin", dtype=np.float32)
output_state = np.fromfile("./output_state.bin", dtype=np.float32)使用CMakeLists.txt进行编译
在howto_deploy目录下创建CMakeLists.txt
cmake_minimum_required(VERSION 3.2)
project(how2delploy C CXX)
SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O3 -Wall -g2 -ggdb")
SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall -fPIC")
set(TVM_ROOT /path/to/tvm)
set(DMLC_CORE ${TVM_ROOT}/3rdparty/dmlc-core)
include_directories(${TVM_ROOT}/include)
include_directories(${DMLC_CORE}/include)
include_directories(${TVM_ROOT}/3rdparty/dlpack/include)
link_directories(${TVM_ROOT}/build/Release)
add_definitions(-DDMLC_USE_LOGGING_LIBRARY=)
add_executable(cpp_deploy_norm cpp_deploy.cc)
target_link_libraries(cpp_deploy_norm ${TVM_ROOT}/build/libtvm_runtime.so)
老四连
mkdir build
cd build
cmake ..
make运行
cd build
./cpp_deploy_norm