在centos6.4 编译64位的hadoop 2.4.1
1.下相关包
yum install svn
yum install autoconfautomake libtool cmake
yum install ncurses-devel
yum install openssl-devel
yum install gcc*
2：下载和解压maven
wget http://apache.fayea.com/apache-mirror/maven/maven-3/3.2.1/binaries/apache-maven-3.2.1-bin.tar.gz
tar xzvf ./apache-maven-3.2.1-bin.tar.gz
mv ./ apache-maven-3.2.1 /usr/local
3 安装 ANT
下载地址http://download.csdn.net/detail/wind520/7705109
压解 apache-ant-1.9.4-bin.tar.gz 到/usr/local目录
# tar -zxvf apache-ant-1.9.4-bin.tar.gz
mv ./ apache-ant-1.9.4 /usr/local
4:配置文件
vi /etc/profile
# /etc/profile
# System wide environment and startup programs, for login setup
# Functions and aliases go in /etc/bashrc
# It's NOT a good idea to change this file unless you know what you
# are doing. It's much better to create a custom.sh shell script in
# /etc/profile.d/ to make custom changes to your environment, as this
# will prevent the need for merging in future updates.
pathmunge () {
case ":${PATH}:" in
*:"$1":*)
;;
*)
if [ "$2" = "after" ] ; then
PATH=$PATH:$1
else
PATH=$1:$PATH
fi
esac
}
if [ -x /usr/bin/id ]; then
if [ -z "$EUID" ]; then
# ksh workaround
EUID=`id -u`
UID=`id -ru`
fi
USER="`id -un`"
LOGNAME=$USER
MAIL="/var/spool/mail/$USER"
fi
# Path manipulation
if [ "$EUID" = "0" ]; then
pathmunge /sbin
pathmunge /usr/sbin
pathmunge /usr/local/sbin
else
pathmunge /usr/local/sbin after
pathmunge /usr/sbin after
pathmunge /sbin after
fi
HOSTNAME=`/bin/hostname 2>/dev/null`
HISTSIZE=1000
if [ "$HISTCONTROL" = "ignorespace" ] ; then
export HISTCONTROL=ignoreboth
else
export HISTCONTROL=ignoredups
fi
export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL
export JAVA_HOME=/usr/jdk1.7.0_45
export CLASSPATH=.:JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH:/usr/local/apache-maven-3.2.1/bin
export ANT_HOME=/usr/local/apache-ant-1.9.4
export PATH=$PATH:$ANT_HOME/bin
# By default, we want umask to get set. This sets it for login shell
# Current threshold for system reserved uid/gids is 200
# You could check uidgid reservation validity in
# /usr/share/doc/setup-*/uidgid file
if [ $UID -gt 199 ] && [ "`id -gn`" = "`id -un`" ]; then
umask 002
else
umask 022
fi
for i in /etc/profile.d/*.sh ; do
if [ -r "$i" ]; then
if [ "${-#*i}" != "$-" ]; then
. "$i"
else
. "$i" >/dev/null 2>&1
fi
fi
done
unset i
unset -f pathmunge
配置文件生效
source /etc/profile
5.安装protobuf
wget https://protobuf.googlecode.com/files/protobuf-2.5.0.tar.gz
由于下载不下来，从其他地方下载protobuf-2.5.0.tar.gz
tar xzvf ./protobuf-2.5.0.tar.gz
cd protobuf-2.5.0
./configure
make
make check
make install
以上操作都用root操作
检查
[jifeng@feng05 ~]$ mvn -v Apache Maven 3.2.1 (ea8b2b07643dbb1b84b6d16e1f08391b666bc1e9; 2014-02-15T01:37:52+08:00) Maven home: /usr/local/apache-maven-3.2.1 Java version: 1.7.0_45, vendor: Oracle Corporation Java home: /usr/jdk1.7.0_45/jre Default locale: zh_CN, platform encoding: UTF-8 OS name: "linux", version: "2.6.32-358.el6.x86_64", arch: "amd64", family: "unix" [jifeng@feng05 ~]$ java -version java version "1.7.0_45" Java(TM) SE Runtime Environment (build 1.7.0_45-b18) Java HotSpot(TM) 64-Bit Server VM (build 24.45-b08, mixed mode) [jifeng@feng05 ~]$ ant -version Apache Ant(TM) version 1.9.4 compiled on April 29 2014 [jifeng@feng05 ~]$
[jifeng@feng05 ~]$ protoc --version libprotoc 2.5.0 [jifeng@feng05 ~]$
6：下载 hadoop-2.4.1代码
下载地址http://download.csdn.net/detail/wind520/7705113
执行 mvn clean install -DskipTests 命令，等待完成(会自动联网下载很多东西
[jifeng@feng05 ~]$ ls hadoop-2.4.1-src hadoop-2.4.1-src.tar.gz jdk-7u45-linux-x64.gz protobuf-2.5.0.tar.gz [jifeng@feng05 ~]$ cd hadoop-2.4.1-src [jifeng@feng05 hadoop-2.4.1-src]$ mvn clean install -DskipTests [INFO] Scanning for projects... [WARNING] [WARNING] Some problems were encountered while building the effective model for org.apache.hadoop:hadoop-project:pom:2.4.1 [WARNING] 'build.plugins.plugin.(groupId:artifactId)' must be unique but found duplicate declaration of plugin org.apache.maven.plugins:maven-enforcer-plugin @ line 1015, column 15 [WARNING]
[INFO] Reactor Summary: [INFO] [INFO] Apache Hadoop Main ................................ SUCCESS [ 20.235 s] [INFO] Apache Hadoop Project POM ......................... SUCCESS [ 8.101 s] [INFO] Apache Hadoop Annotations ......................... SUCCESS [ 4.639 s] [INFO] Apache Hadoop Project Dist POM .................... SUCCESS [ 0.840 s] [INFO] Apache Hadoop Assemblies .......................... SUCCESS [ 0.279 s] [INFO] Apache Hadoop Maven Plugins ....................... SUCCESS [ 4.143 s] [INFO] Apache Hadoop MiniKDC ............................. SUCCESS [ 3.495 s] [INFO] Apache Hadoop Auth ................................ SUCCESS [ 3.161 s] [INFO] Apache Hadoop Auth Examples ....................... SUCCESS [ 2.117 s] [INFO] Apache Hadoop Common .............................. SUCCESS [01:58 min] [INFO] Apache Hadoop NFS ................................. SUCCESS [ 24.001 s] [INFO] Apache Hadoop Common Project ...................... SUCCESS [ 0.028 s] [INFO] Apache Hadoop HDFS ................................ SUCCESS [01:25 min] [INFO] Apache Hadoop HttpFS .............................. SUCCESS [ 8.315 s] [INFO] Apache Hadoop HDFS BookKeeper Journal ............. SUCCESS [02:23 min] [INFO] Apache Hadoop HDFS-NFS ............................ SUCCESS [ 1.240 s] [INFO] Apache Hadoop HDFS Project ........................ SUCCESS [ 0.082 s] [INFO] hadoop-yarn ....................................... SUCCESS [ 0.123 s] [INFO] hadoop-yarn-api ................................... SUCCESS [ 3.832 s] [INFO] hadoop-yarn-common ................................ SUCCESS [ 35.130 s] [INFO] hadoop-yarn-server ................................ SUCCESS [ 0.026 s] [INFO] hadoop-yarn-server-common ......................... SUCCESS [ 0.791 s] [INFO] hadoop-yarn-server-nodemanager .................... SUCCESS [01:52 min] [INFO] hadoop-yarn-server-web-proxy ...................... SUCCESS [ 0.682 s] [INFO] hadoop-yarn-server-applicationhistoryservice ...... SUCCESS [01:06 min] [INFO] hadoop-yarn-server-resourcemanager ................ SUCCESS [ 2.334 s] [INFO] hadoop-yarn-server-tests .......................... SUCCESS [ 0.798 s] [INFO] hadoop-yarn-client ................................ SUCCESS [ 1.615 s] [INFO] hadoop-yarn-applications .......................... SUCCESS [ 0.033 s] [INFO] hadoop-yarn-applications-distributedshell ......... SUCCESS [ 0.358 s] [INFO] hadoop-yarn-applications-unmanaged-am-launcher .... SUCCESS [ 0.216 s] [INFO] hadoop-yarn-site .................................. SUCCESS [ 0.072 s] [INFO] hadoop-yarn-project ............................... SUCCESS [ 0.080 s] [INFO] hadoop-mapreduce-client ........................... SUCCESS [ 0.081 s] [INFO] hadoop-mapreduce-client-core ...................... SUCCESS [ 4.187 s] [INFO] hadoop-mapreduce-client-common .................... SUCCESS [ 1.690 s] [INFO] hadoop-mapreduce-client-shuffle ................... SUCCESS [ 0.423 s] [INFO] hadoop-mapreduce-client-app ....................... SUCCESS [ 2.596 s] [INFO] hadoop-mapreduce-client-hs ........................ SUCCESS [ 1.440 s] [INFO] hadoop-mapreduce-client-jobclient ................. SUCCESS [ 15.030 s] [INFO] hadoop-mapreduce-client-hs-plugins ................ SUCCESS [ 0.315 s] [INFO] Apache Hadoop MapReduce Examples .................. SUCCESS [ 0.751 s] [INFO] hadoop-mapreduce .................................. SUCCESS [ 0.099 s] [INFO] Apache Hadoop MapReduce Streaming ................. SUCCESS [ 5.560 s] [INFO] Apache Hadoop Distributed Copy .................... SUCCESS [ 2.346 s] [INFO] Apache Hadoop Archives ............................ SUCCESS [ 0.465 s] [INFO] Apache Hadoop Rumen ............................... SUCCESS [ 0.917 s] [INFO] Apache Hadoop Gridmix ............................. SUCCESS [ 0.609 s] [INFO] Apache Hadoop Data Join ........................... SUCCESS [ 0.250 s] [INFO] Apache Hadoop Extras .............................. SUCCESS [ 0.395 s] [INFO] Apache Hadoop Pipes ............................... SUCCESS [ 0.047 s] [INFO] Apache Hadoop OpenStack support ................... SUCCESS [ 1.052 s] [INFO] Apache Hadoop Client .............................. SUCCESS [ 0.443 s] [INFO] Apache Hadoop Mini-Cluster ........................ SUCCESS [ 0.209 s] [INFO] Apache Hadoop Scheduler Load Simulator ............ SUCCESS [ 8.011 s] [INFO] Apache Hadoop Tools Dist .......................... SUCCESS [ 0.252 s] [INFO] Apache Hadoop Tools ............................... SUCCESS [ 0.025 s] [INFO] Apache Hadoop Distribution ........................ SUCCESS [ 0.066 s] [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESS [INFO] ------------------------------------------------------------------------ [INFO] Total time: 11:50 min [INFO] Finished at: 2014-08-01T17:15:04+08:00 [INFO] Final Memory: 137M/405M [INFO] ------------------------------------------------------------------------
7：开始编译
[jifeng@feng05 hadoop-2.4.1-src]$ mvn package -Pdist,native -DskipTests -Dtar
[jifeng@feng05 hadoop-2.4.1-src]$ mvn package -Pdist,native -DskipTests -Dtar [INFO] Scanning for projects... [WARNING] [WARNING] Some problems were encountered while building the effective model for org.apache.hadoop:hadoop-project:pom:2.4.1 [WARNING] 'build.plugins.plugin.(groupId:artifactId)' must be unique but found duplicate declaration of plugin org.apache.maven.plugins:maven-enforcer-plugin @ line 1015, column 15 [WARNING]
编译成功
[INFO] ------------------------------------------------------------------------ [INFO] Reactor Summary: [INFO] [INFO] Apache Hadoop Main ................................ SUCCESS [ 2.158 s] [INFO] Apache Hadoop Project POM ......................... SUCCESS [ 2.495 s] [INFO] Apache Hadoop Annotations ......................... SUCCESS [ 4.365 s] [INFO] Apache Hadoop Assemblies .......................... SUCCESS [ 0.321 s] [INFO] Apache Hadoop Project Dist POM .................... SUCCESS [ 2.327 s] [INFO] Apache Hadoop Maven Plugins ....................... SUCCESS [ 4.388 s] [INFO] Apache Hadoop MiniKDC ............................. SUCCESS [ 3.563 s] [INFO] Apache Hadoop Auth ................................ SUCCESS [ 4.287 s] [INFO] Apache Hadoop Auth Examples ....................... SUCCESS [ 2.544 s] [INFO] Apache Hadoop Common .............................. SUCCESS [01:56 min] [INFO] Apache Hadoop NFS ................................. SUCCESS [ 8.552 s] [INFO] Apache Hadoop Common Project ...................... SUCCESS [ 0.071 s] [INFO] Apache Hadoop HDFS ................................ SUCCESS [02:40 min] [INFO] Apache Hadoop HttpFS .............................. SUCCESS [01:28 min] [INFO] Apache Hadoop HDFS BookKeeper Journal ............. SUCCESS [ 42.631 s] [INFO] Apache Hadoop HDFS-NFS ............................ SUCCESS [ 5.748 s] [INFO] Apache Hadoop HDFS Project ........................ SUCCESS [ 0.052 s] [INFO] hadoop-yarn ....................................... SUCCESS [ 0.044 s] [INFO] hadoop-yarn-api ................................... SUCCESS [01:13 min] [INFO] hadoop-yarn-common ................................ SUCCESS [ 44.626 s] [INFO] hadoop-yarn-server ................................ SUCCESS [ 0.347 s] [INFO] hadoop-yarn-server-common ......................... SUCCESS [ 12.337 s] [INFO] hadoop-yarn-server-nodemanager .................... SUCCESS [ 21.867 s] [INFO] hadoop-yarn-server-web-proxy ...................... SUCCESS [ 3.344 s] [INFO] hadoop-yarn-server-applicationhistoryservice ...... SUCCESS [ 7.377 s] [INFO] hadoop-yarn-server-resourcemanager ................ SUCCESS [ 16.961 s] [INFO] hadoop-yarn-server-tests .......................... SUCCESS [ 0.581 s] [INFO] hadoop-yarn-client ................................ SUCCESS [ 6.511 s] [INFO] hadoop-yarn-applications .......................... SUCCESS [ 0.080 s] [INFO] hadoop-yarn-applications-distributedshell ......... SUCCESS [ 3.672 s] [INFO] hadoop-yarn-applications-unmanaged-am-launcher .... SUCCESS [ 1.866 s] [INFO] hadoop-yarn-site .................................. SUCCESS [ 0.061 s] [INFO] hadoop-yarn-project ............................... SUCCESS [ 5.057 s] [INFO] hadoop-mapreduce-client ........................... SUCCESS [ 0.150 s] [INFO] hadoop-mapreduce-client-core ...................... SUCCESS [ 25.144 s] [INFO] hadoop-mapreduce-client-common .................... SUCCESS [ 20.414 s] [INFO] hadoop-mapreduce-client-shuffle ................... SUCCESS [ 3.011 s] [INFO] hadoop-mapreduce-client-app ....................... SUCCESS [ 11.590 s] [INFO] hadoop-mapreduce-client-hs ........................ SUCCESS [ 11.404 s] [INFO] hadoop-mapreduce-client-jobclient ................. SUCCESS [ 7.434 s] [INFO] hadoop-mapreduce-client-hs-plugins ................ SUCCESS [ 1.847 s] [INFO] Apache Hadoop MapReduce Examples .................. SUCCESS [ 7.132 s] [INFO] hadoop-mapreduce .................................. SUCCESS [ 6.131 s] [INFO] Apache Hadoop MapReduce Streaming ................. SUCCESS [ 6.530 s] [INFO] Apache Hadoop Distributed Copy .................... SUCCESS [ 8.911 s] [INFO] Apache Hadoop Archives ............................ SUCCESS [ 2.109 s] [INFO] Apache Hadoop Rumen ............................... SUCCESS [ 8.018 s] [INFO] Apache Hadoop Gridmix ............................. SUCCESS [ 5.438 s] [INFO] Apache Hadoop Data Join ........................... SUCCESS [ 3.889 s] [INFO] Apache Hadoop Extras .............................. SUCCESS [ 3.361 s] [INFO] Apache Hadoop Pipes ............................... SUCCESS [ 7.744 s] [INFO] Apache Hadoop OpenStack support ................... SUCCESS [ 6.382 s] [INFO] Apache Hadoop Client .............................. SUCCESS [ 6.479 s] [INFO] Apache Hadoop Mini-Cluster ........................ SUCCESS [ 0.388 s] [INFO] Apache Hadoop Scheduler Load Simulator ............ SUCCESS [ 5.011 s] [INFO] Apache Hadoop Tools Dist .......................... SUCCESS [ 5.064 s] [INFO] Apache Hadoop Tools ............................... SUCCESS [ 0.143 s] [INFO] Apache Hadoop Distribution ........................ SUCCESS [ 26.096 s] [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESS [INFO] ------------------------------------------------------------------------ [INFO] Total time: 14:00 min [INFO] Finished at: 2014-08-01T17:41:46+08:00 [INFO] Final Memory: 85M/331M [INFO] ------------------------------------------------------------------------
8：检查64位文件
[jifeng@feng05 hadoop-2.4.1-src]$ cd hadoop-dist/target/hadoop-2.4.1/lib/native [jifeng@feng05 native]$ ls libhadoop.a libhadoop.so libhadooputils.a libhdfs.so libhadooppipes.a libhadoop.so.1.0.0 libhdfs.a libhdfs.so.0.0.0 [jifeng@feng05 native]$ file * libhadoop.a: current ar archive libhadooppipes.a: current ar archive libhadoop.so: symbolic link to `libhadoop.so.1.0.0' libhadoop.so.1.0.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped libhadooputils.a: current ar archive libhdfs.a: current ar archive libhdfs.so: symbolic link to `libhdfs.so.0.0.0' libhdfs.so.0.0.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped编译成果： http://download.csdn.net/detail/wind520/7708355