嵌入式深度学习之mxnet交叉编译Arm Linux

参考文档

1. http://mxnet.incubator.apache.org/install/index.html mxnet官方install手册

本文档涉及到的目标硬件为全志H8(8核Cortex-A7，Armv7架构)，但是对其他Arm芯片也有一定的借鉴意义，只需要更换交叉编译链即可。

开发环境介绍

• 主机操作系统：Ubuntu14.04 64位
• 目标平台：CQA83t 全志H8
• 交叉工具链：arm-linux-gnueabihf，gcc4.9.2
• mxnet版本：20180124-master

注意：mxnet的最低GCC版本为4.8.0

设置交叉编译链

# 在/etc/bash.bashrc的最后增加如下指令
# Allwinner H8 CQA83t cross compiler
export ARCH=arm
export PATH=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/:$PATH
export CROSS_COMPILE=arm-linux-gnueabihf-
export CC=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc    
export CXX=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-g++    
export LD=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ld
export AR=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ar
export AS=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-as
export RANLIB=/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-ranlib
# 修改完成之后需要重启命令行才能生效
# 你可以通过如下指令来确认交叉编译链是否已经设置好
echo $CC 
# 当显示/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc时表示交叉编译链
# 已经设置好
# 当需要更换为本机编译时屏蔽上面的指令即可

下载依赖库

依赖库	版本	下载地址
openblas	0.2.20	https://github.com/xianyi/OpenBLAS

编译OpenBLAS

编译

cd OpenBLAS-0.2.20
# 编译单独的静态库版本
make TARGET=ARMV7 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=arm-linux-gnueabihf- BINARY=32 NOFORTRAN=1 libs -j8
# 编译动态库版本
make TARGET=ARMV7 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=arm-linux-gnueabihf- BINARY=32 NOFORTRAN=1 libs shared

# 另附上ARMV8芯片的编译命令以供参考
make TARGET=ARMV8 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=aarch64-linux-gnu- BINARY=64 NOFORTRAN=1 libs -j8
make TARGET=ARMV8 HOSTCC=gcc CROSS=1 CROSS_SUFFIX=aarch64-linux-gnu- BINARY=64 NOFORTRAN=1 libs shared -j8

安装

# PREFIX使用您自己的自定义安装目录
make PREFIX=/usr/local/H8/openBLAS-0.2.20 install

编译mxnet

修改CmakeLists.txt

修改
# ---[ LAPack
if(USE_LAPACK)
  add_definitions(-DMXNET_USE_LAPACK=1)
  list(APPEND mxnet_LINKER_LIBS lapack)
else(USE_LAPACK)
  Workaround for Windows until using new Jenkinsfile.
  if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
    add_definitions(-DMXNET_USE_LAPACK=1)
  endif()
endif()
为
# ---[ LAPack
if(USE_LAPACK)
  add_definitions(-DMXNET_USE_LAPACK=1)
  list(APPEND mxnet_LINKER_LIBS lapack)
else(USE_LAPACK)
  # Workaround for Windows until using new Jenkinsfile.
  # if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
  #   add_definitions(-DMXNET_USE_LAPACK=1)
  # endif()
endif()

# 这里屏蔽add_definitions(-DMXNET_USE_LAPACK=1)是因为这里的添加是Windows专用的,针对Linux需要屏蔽

修改tests/CmakeLists.txt

修改
if(NOT MSVC)
  set(UNITTEST_STATIC_LINK ON)
endif()
为
if(NOT MSVC)
  set(UNITTEST_STATIC_LINK OFF)
endif()

修改
if(UNITTEST_STATIC_LINK)
    target_link_libraries(${PROJECT_NAME}_unit_tests
      ${GTEST_LIBRARY}
      ${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE}
      dmlc
      ${mxnet_LINKER_LIBS}
      ${pslite_LINKER_LIBS}
      )
  else()
    target_link_libraries(${PROJECT_NAME}_unit_tests
      ${GTEST_LIBRARY}
      dmlc
      ${nnvm_LINKER_LIBS}
      ${mxnet_LINKER_LIBS}
      mxnet
      ${pslite_LINKER_LIBS}
      )
  endif()
为
if(UNITTEST_STATIC_LINK)
    target_link_libraries(${PROJECT_NAME}_unit_tests
      ${GTEST_LIBRARY}
      ${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE}
      dmlc
      ${mxnet_LINKER_LIBS}
      ${pslite_LINKER_LIBS}
      )
  else()
    target_link_libraries(${PROJECT_NAME}_unit_tests
      ${GTEST_LIBRARY}
      dmlc
      ${nnvm_LINKER_LIBS}
      ${mxnet_LINKER_LIBS}
      mxnet
      ${pslite_LINKER_LIBS}
      rt
      )
  endif()

新建cmake Build目录

cd incubator-mxnet
mkdir _install

打开cmake-gui进行交叉编译链配置

注意：mxnet要求cmake版本需要高于3.0.2

选择Specify options for cross-compiling，进入交叉编译链设置界面

# 设置交叉编译相关参数
# 本文使用的是NVIDIA提供的交叉工具链，存放位置为/opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/
# (更改为你正在使用的目录)
# 设置Operating System 为 arm-linux 
# 设置C编译器为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-gcc
# (更改为你正在使用的gcc)
# 设置C++编译器为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/bin/arm-linux-gnueabihf-g++ 
# 设置System Root 为 /opt/toolchain/gcc-linaro-arm-linux-gnueabihf-4.9-2014.09_linux/arm-linux-gnueabihf/sysroot

点击Configure生成配置选项，并且打开Advancd选项

此时Cmake会显示CUDA错误，暂时先不用管，后面修改设置后错误会消失

修改设置选项：

# 去掉勾选为FLASE 勾选为TRUE
USE_CUDA                          # FALSE   
USE_CUDNN                         # FALSE
USE_GPERFTOOLS                    # FALSE
USE_JEMALLOC                      # FALSE
USE_LAPACK                        # FALSE
USE_OPENCV                        # FALSE
USE_OPENMP                        # TRUE 多核ARM建议开启openmp

点击Configure

再次修改设置选项：

# 指定安装类型和目录
CMAKE_BUILD_TYPE                  Release
CMAKE_INSTALL_PREFIX              /usr/local/H8/mxnet-20180122

CMAKE_C_FLAGS                     -mfpu=neon -march=armv7-a
CMAKE_CXX_FLAGS                   -mfpu=neon -march=armv7-a
BLAS                              Open
# 指定OpenBLAS头文件和库文件路径(填写之前交叉编译OpenBLAS的安装目录)
OpenBLAS_INCLUDE_DIR              /usr/local/H8/openBLAS-0.2.20/include
OpenBLAS_LIBRARY                  /usr/local/H8/openBLAS-0.2.20/lib/libopenblas.so

点击Configure

点击Generate生成Makefile

进入_install目录&编译

cd _install
make -j8 
make install

查看libmxnet.so的依赖库

arm-linux-gnueabihf-readelf -d /usr/local/H8/mxnet-20180122/lib/libmxnet.so
Dynamic section at offset 0x13fde68 contains 33 entries:
  Tag        Type                         Name/Value
 0x00000001 (NEEDED)                     Shared library: [libopenblas.so.0]
 0x00000001 (NEEDED)                     Shared library: [3rdparty/openmp/runtime/src/libomp.so]
 0x00000001 (NEEDED)                     Shared library: [libpthread.so.0]
 0x00000001 (NEEDED)                     Shared library: [libdl.so.2]
 0x00000001 (NEEDED)                     Shared library: [libstdc++.so.6]
 0x00000001 (NEEDED)                     Shared library: [libm.so.6]
 0x00000001 (NEEDED)                     Shared library: [libgomp.so.1]
 0x00000001 (NEEDED)                     Shared library: [libgcc_s.so.1]
 0x00000001 (NEEDED)                     Shared library: [libc.so.6]
 0x00000001 (NEEDED)                     Shared library: [ld-linux-armhf.so.3]

从上面的依赖库可以看到3rdparty/openmp/runtime/src/libomp.so这个相对路径，这样并不利于最终的部署测试

可以尝试改动Cmake生成的Makefile来解决

修改_install/CMakeFiles/mxnet.dir/link.txt

将3rdparty/openmp/runtime/src/libomp.so
修改为-L/usr/local/H8/mxnet-20180122/lib/ -lomp
# /usr/local/H8/mxnet-20180122/lib/ 即是之前的安装目录

重新进行一次编译&安装

make -j8 
make install