mac系统编译3.2.1版本hadoop

缘由

编译hadoop主要是为了解决安装hadoop缺少lib/native，发生的“WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform using builtin-java classes where applicable”警告，这个警告虽然不影响hadoop的正常运行
中间想过放弃编译直接下载安装包，用其中的lib/native，发现mac下不能用.so的包。

中间走的弯路，没有效果，是进到hadoop-hdfs-native-client中的src目录下运行的
cmake -DJVM_ARCH_DATA_MODEL=64 -DOPENSSL_ROOT_DIR=/usr/local/Cellar/[email protected]/1.1.1g  -DOPENSSL_LIBRARIES=/usr/local/Cellar/[email protected]/1.1.1g/lib -DOPENSSL_INCLUDE_DIR=/usr/local/Cellar/[email protected]/1.1.1g/include -DPROTOBUF_INCLUDE_DIR=/usr/local/Cellar/protobuf/include -DPROTOBUF_LIBRARY=/usr/local/Cellar/protobuf/lib/libprotoc.dylib

步骤

下载hadoop源码

从官网上下载：下载地址
解压后注意阅读BUILDING.txt文件
Requirements:

• Unix System
• JDK 1.8
• Maven 3.3 or later
• ProtocolBuffer 2.5.0
• CMake 3.1 or newer (if compiling native code)
• Zlib devel (if compiling native code)
• Cyrus SASL devel (if compiling native code)
• One of the compilers that support thread_local storage: GCC 4.8.1 or later, Visual Studio,
  Clang (community version), Clang (version for iOS 9 and later) (if compiling native code)
• openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
• Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
• Doxygen ( if compiling libhdfspp and generating the documents )
• Internet connection for first build (to fetch all Maven and Hadoop dependencies)
• python (for releasedocs)
• bats (for shell code testing)
• Node.js / bower / Ember-cli (for YARN UI v2 building)

安装maven

brew install maven

#或者下载maven的包解压到/usr/local/maven，在bash_profile中配置

export MAVEN_HOME=/usr/local/maven
export PATH=$PATH:$MAVEN_HOME/bin


#配置settings
<mirrors>
      <mirror>
        <id>alimaven</id>
        <name>aliyun maven</name>
        <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
        <mirrorOf>central</mirrorOf>        
      </mirror>
      <mirror>
        <id>repo1</id>
        <mirrorOf>central</mirrorOf>
        <name>repo1 maven</name>
        <url>http://repo1.maven.org/maven2/</url>
      </mirror>
     <mirror>
         <id>repo2</id>
         <mirrorOf>central</mirrorOf>
         <name>repo2 maven</name>
         <url>http://repo2.maven.org/maven2/</url>
    </mirror>
</mirrors>

安装openssl

brew install openssl

#配置bash_profile，添加OPENSSL_ROOT_DIR，OPENSSL_INCLUDE_DIR
export OPENSSL_ROOT_DIR=/usr/local/Cellar/openssl/1.0.2k
export OPENSSL_INCLUDE_DIR=/usr/local/Cellar/openssl/1.0.2k/include

可能可以修改mac默认的openssl，不建议尝试

安装protobuf-2.5.0

#下载protobuf-2.5.0源码包，并编译，安装包百度搜索一下，不建议./configure --prefix=/usr/local/Cellar/protobuf，配置了环境变量也没有效果，mvn打包时cmake还是会提示PROTOBUF的根目录找不到
#export PROTOBUF=/usr/local/Cellar/protobuf
#export PATH=$PROTOBUF/bin:$PATH

cd protobuf-2.5.0
./configure
make
make check
make install
protoc --version

安装cmake

下载cmake3.1.0或以上的安装包，3.1.0以上的版本好像有问题，打包时提示找不到openssl，这里采用3.1.0版本

#编译并安装
./configure && make && make install

cmake --version

#***注意/usr/local/Cellar/cmake/share/cmake-3.1/Modules目录下的FindOpenSSL.cmake文件要修改一下，解析openssl的版本号的时候出bug，#define后面带空格或制表符的
if (OPENSSL_INCLUDE_DIR)
  if (_OPENSSL_VERSION)
    set(OPENSSL_VERSION "${_OPENSSL_VERSION}")
  elseif(OPENSSL_INCLUDE_DIR AND EXISTS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h")
    file(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h" openssl_version_str
         REGEX "^#[\t ]+define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])+.*")

    # The version number is encoded as 0xMNNFFPPS: major minor fix patch status
    # The status gives if this is a developer or prerelease and is ignored here.
    # Major, minor, and fix directly translate into the version numbers shown in
    # the string. The patch field translates to the single character suffix that
    # indicates the bug fix state, which 00 -> nothing, 01 -> a, 02 -> b and so
    # on.
   
    string(REGEX REPLACE "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F][0-9a-fA-F])([0-9a-fA-F]).*$"
           "\\1;\\2;\\3;\\4;\\5" OPENSSL_VERSION_LIST "${openssl_version_str}")
    message("================ ${OPENSSL_INCLUDE_DIR} ${openssl_version_str}")       
    list(GET OPENSSL_VERSION_LIST 0 OPENSSL_VERSION_MAJOR)
    list(GET OPENSSL_VERSION_LIST 1 OPENSSL_VERSION_MINOR)
    from_hex("${OPENSSL_VERSION_MINOR}" OPENSSL_VERSION_MINOR)
    list(GET OPENSSL_VERSION_LIST 2 OPENSSL_VERSION_FIX)
    from_hex("${OPENSSL_VERSION_FIX}" OPENSSL_VERSION_FIX)
    list(GET OPENSSL_VERSION_LIST 3 OPENSSL_VERSION_PATCH)

    if (NOT OPENSSL_VERSION_PATCH STREQUAL "00")
      from_hex("${OPENSSL_VERSION_PATCH}" _tmp)
      # 96 is the ASCII code of 'a' minus 1
      math(EXPR OPENSSL_VERSION_PATCH_ASCII "${_tmp} + 96")
      unset(_tmp)
      # Once anyone knows how OpenSSL would call the patch versions beyond 'z'
      # this should be updated to handle that, too. This has not happened yet
      # so it is simply ignored here for now.
      string(ASCII "${OPENSSL_VERSION_PATCH_ASCII}" OPENSSL_VERSION_PATCH_STRING)
    endif ()

    set(OPENSSL_VERSION "${OPENSSL_VERSION_MAJOR}.${OPENSSL_VERSION_MINOR}.${OPENSSL_VERSION_FIX}${OPENSSL_VERSION_PATCH_STRING}")
  endif ()
endif ()

#配置bash_profile
export CMAKE_HOME=/usr/local/Cellar/cmake
export PATH=$PATH:$CMAKE_HOME/bin

安装zlib

brew install zlib

安装gsasl

brew install gsasl

安装Doxygen

brew install doxygen

可选安装zstd

brew install zstd

可选安装snappy

brew install snappy

安装后打包编译出问题了，编译到nativetask发生了错误“error: unknown type name ‘constexpr’”

可以不用brew安装，到github下载源码编译,地址

mkdir build
cd build && cmake ../ && make

猜测出错的原因是C++环境的问题，可能需要安装gcc，安装后没有效果

后面发现，“constexpr”需要在c++11环境下，我们要修改hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src里面的CMakeLists.txt，在include(HadoopJNI)后面添加如下代码

include(CheckFunctionExists)
include(CheckIncludeFiles)
include(CheckCXXCompilerFlag) 
CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 
CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) 
if(COMPILER_SUPPORTS_CXX11) 
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 
elseif(COMPILER_SUPPORTS_CXX0X) 
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") 
else() 
    message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 
endif()

重新打包编译hadoop，发现报错“error: invalid suffix on literal; C++11 requires a space between literal and identifier [-Wreserved-user-defined-literal]”

hadoop-mapreduce-client-nativetask/src/main/native中所有的*.cc中出现的"%“PRId64、”%"PRIu64，需要改成引号和P之间要加空格，否则c++11编译不过，改完之后，编译hadoop，lib/native下多了

执行hadoop checknative，snappy通过

可选安装isa-l

下载isa-l源码包地址，解压

cd isa-l-2.28.0

#执行创建configure

autoreconf --install --symlink -f

./configure --prefix=/usr/local/Cellar/isa-l --libdir=/usr/local/Cellar/isa-l/lib  AS=yasm --target=darwin

make

make install

cd /usr/local/lib
ln -s /usr/local/Cellar/isa-l/lib/libisal.2.dylib libisal.2.dylib
ln -s /usr/local/Cellar/isa-l/lib/libisal.a libisal.a
ln -s /usr/local/Cellar/isa-l/lib/libisal.dylib libisal.dylib
ln -s /usr/local/Cellar/isa-l/lib/libisal.la libisal.la

cd /usr/local/lib/pkgconfig
ln -s /usr/local/Cellar/isa-l/lib/pkgconfig/libisal.pc libisal.pc

打包编译hadoop

cd hadoop-3.2.1-src
#需要指定openssl.prefix否则默认用mac系统自带的openssl，另外建议maven的中央仓库mirror配置为阿里的
#配置pom.xml，原来的在国内速度很慢改成aliyun的
<repositories>
    <repository>
      <id>${distMgmtSnapshotsId}</id>
      <name>${distMgmtSnapshotsName}</name>
      <url>${distMgmtSnapshotsUrl}</url>
    </repository>
    <repository>
            <id>alimaven</id>
            <name>aliyun maven</name>
            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
    </repository>
</repositories>
 #不添加可选依赖 
mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip -Dopenssl.prefix=/usr/local/Cellar/openssl/1.0.2k -e

#添加snappy可选依赖，-Dbundle.snappy可以去掉
mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip -Dopenssl.prefix=/usr/local/Cellar/openssl/1.0.2k -Drequire.snappy -Dsnappy.lib=/usr/local/Cellar/snappy/1.1.8/lib -Dbundle.snappy -e 

#添加isa-l可选依赖，-Dbundle.isal可以去掉
mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip -Dopenssl.prefix=/usr/local/Cellar/openssl/1.0.2k -Drequire.snappy -Dsnappy.lib=/usr/local/Cellar/snappy/1.1.8/lib -Dbundle.snappy -Drequire.isal -Disal.prefix=/usr/local/Cellar/isa-l -Disal.lib=/usr/local/Cellar/isa-l/lib -Dbundle.isal -e

静静地等待打包，时间较长，中途可能会失败，失败了需要mvn clean一下

检查lib/native

将编译后的lib/native拷贝到libexec目录，然后vim修改.bash_profile

#修改.bash_profile
export HADOOP_HOME=/usr/local/Cellar/hadoop/3.2.1/libexec
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
#可以选择将HADOOP_HOME的bin目录配置到PATH

source ~/.bash_profile

#重新打开terminal

进入安装的hadoop的libexec/etc/hadoop目录，修改hadoop-env.sh文件，在对应的位置添加

#设置java_home
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_121.jdk/Contents/Home
#设置lib/native的位置
export HADOOP_OPTS="-Djava.library.path=$HADOOP_COMMON_LIB_NATIVE_DIR"
#设置日志的级别
export HADOOP_ROOT_LOGGER=DEBUG,console

进入sbin目录执行hadoop checknative

pwd
#/usr/local/Cellar/hadoop/3.2.1/sbin
hadoop checknative

#2020-06-19 15:24:44,956 DEBUG util.NativeCodeLoader: Trying to load the custom-built native-hadoop library...
#2020-06-19 15:24:44,961 DEBUG util.NativeCodeLoader: Loaded the native-hadoop library
#2020-06-19 15:24:45,133 WARN bzip2.Bzip2Factory: Failed to load/initialize native-bzip2 library system-native, will use pure-Java version
#2020-06-19 15:24:45,139 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
#2020-06-19 15:24:45,149 WARN erasurecode.ErasureCodeNative: ISA-L support is not available in your platform... using builtin-java codec where applicable
#2020-06-19 15:24:45,315 DEBUG util.Shell: setsid is not available on this machine. So not using it.
#2020-06-19 15:24:45,315 DEBUG util.Shell: setsid exited with exit code 0
#Native library checking:
#hadoop:  true /usr/local/Cellar/hadoop/3.2.1/libexec/lib/native/libhadoop.dylib
#zlib:    true /usr/lib/libz.1.dylib
#zstd  :  true /usr/local/Cellar/zstd/1.4.5/lib/libzstd.1.4.5.dylib
#snappy:  false 
#lz4:     true revision:10301
#bzip2:   false 
#openssl: true /usr/lib/libcrypto.35.dylib
#ISA-L:   false libhadoop was built without ISA-L support

参考资料

http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/NativeLibraries.html
https://blog.csdn.net/qq_14811559/article/details/80308434
https://www.cnblogs.com/edan/p/9968310.html
https://stackoverflow.com/questions/29792009/cmake-osx-mac-openssl-brew
https://blog.csdn.net/plifemao/article/details/86711771