Hadoop集群搭建

下载hadoop3.0.0安装包

在usr下创建目录hadoop:
[root@nn1 usr]# mkdir hadoop

进入新目录(hadoop) 并把安装包复制过来:
[root@nn1 hadoop]# mv /home/admin/software/hadoop-3.0.0-src.tar.gz ./

解压:
[root@nn1 hadoop]# tar -zxvf hadoop-3.0.0-src.tar.gz

得到hadoop-3.0.0目录  并进入
[root@nn1 hadoop]# cd hadoop-3.0.0/

在/usr/hadoop/hadoop-3.0.0/目录下，建立tmp、hdfs/name、hdfs/data目录
[root@nn1 hadoop-3.0.0]# mkdir hdfs
[root@nn1 hadoop-3.0.0]# mkdir tmp
[root@nn1 hadoop-3.0.0]# mkdir hdfs/data
[root@nn1 hadoop-3.0.0]# mkdir hdfs/name

设置环境变量:
# set hadoop path
export HADOOP_HOME=/usr/hadoop/hadoop-3.0.0
export PATH=$PATH:$HADOOP_HOME/bin

使环境变量生效
[root@nn1 hadoop-3.0.0]# source /etc/profile

设置hadoop文件

hadoop-env.sh

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/hadoop-env.sh

#
# The java implementation to use.  
#export JAVA_HOME=${JAVA_HOME}  
export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_161

yarn-env.sh

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/yarn-site.xml 

#The java implementation to usr  
export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_161

core-site.xml

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/core-site.xml

    fs.defaultFS
    hdfs://192.168.0.240:9000



    hadoop.tmp.dir
    /usr/hadoop/hadoop-3.0.0/tmp

hdfs-site.xml

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/hdfs-site.xml

    dfs.nameservices
    myhdfs




    dfs.ha.namenode.myhdfs
    nn1,nn2




    dfs.namenode.rpc-address.myhdfs.nn1
    192.168.0.240:9000




    dfs.namenode.rpc-address.myhdfs.nn2
    192.168.0.241:9000




    dfs.namenode.http-address.myhdfs.nn1
    192.168.0.240:50070




    dfs.namenode.http-address.myhdfs.nn2
    192.168.0.241:50070



    dfs.namenode.http-address
    192.168.0.240:50070



    dfs.namenode.secondary.http-address
    192.168.0.241:50070




    dfs.namenode.name.dir
    /usr/hadoop/hadoop-3.0.0/hdfs/name




    dfs.datanode.data.dir
    /usr/hadoop/hadoop-3.0.0/hdfs/data




    dfs.replication
    2

mapred-site.xml

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/mapred-site.xml

        
        mapreduce.framework.name
        yarn


         
         mapred.job.tracker.http.address
         192.168.0.240:50030


         
         mapred.task.tracker.http.address
         192.168.0.240:50060

yarn-site.xml

[root@nn1 hadoop-3.0.0]# vim /usr/hadoop/hadoop-3.0.0/etc/hadoop/yarn-site.xml

 
    
        yarn.resourcemanager.cluster-id
        nn1
    

     
    
        yarn.resourcemanager.ha.enabled
        true
    
    
     
    
        yarn.resourcemanager.zk-state-store.parent-path
        /rmstore/nn1
    

     
    
        yarn.resourcemanager.ha.automatic-failover.enabled
        true
    

    
    
        yarn.resourcemanager.ha.automatic-failover.embedded
        true
    

    
        yarn.resourcemanager.ha.automatic-failover.zk-nn1-path
        /yarn-leader-election
    

    
    
        yarn.resourcemanager.ha.rm-ids
        rm1,rm2
    

    
    
        yarn.resourcemanager.hostname.rm1
        nn1
    

    
        yarn.resourcemanager.hostname.rm2
        dn1
    

    
        yarn.resourcemanager.recovery.enabled
        false
    

    
        yarn.resourcemanager.store.class
        org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
    

    
    
        yarn.resourcemanager.zk-address
        nn1:2181,dn1:2182,dn2:2183
    

    
        yarn.resourcemanager.webapp.address.rm1
        nn1:8088
    

    
        yarn.resourcemanager.webapp.address.rm2
        dn1:8088
    
    
    
        yarn.nodemanager.aux-services
        mapreduce_shuffle
        true
    

启动之前:

添加
# vim sbin/start-dfs.sh
# vim sbin/stop-dfs.sh

内容为:
HDFS_DATANODE_USER=root  
HADOOP_SECURE_DN_USER=hdfs  
HDFS_NAMENODE_USER=root  
HDFS_SECONDARYNAMENODE_USER=root  

添加
# vim sbin/start-yarn.sh 
# vim sbin/stop-yarn.sh 

内容为:
YARN_RESOURCEMANAGER_USER=root  
HADOOP_SECURE_DN_USER=yarn  
YARN_NODEMANAGER_USER=root 

将整个hadoop3.0.0文件复制到两台分机上

scp -r /usr/hadoop/hadoop-3.0.0/ [email protected]:/usr/hadoop/
scp -r /usr/hadoop/hadoop-3.0.0/ [email protected]:/usr/hadoop/

最后

./sbin/start-all.sh (全部启动)