Hadoop+Hbase+Spark集群配置—Hadoop HA安装



解压、改名
tar -zxvf hadoop-2.6.0.tar.gz
mv hadoop-2.6.0 hadoop
验证hadoop安装成功
hadoop version

修改hadoop配置文件
[1]
vi /usr/hadoop/hadoop/etc/hadoop/core-site.xml
###############################################



 
 
    fs.trash.interval
    1440
 

 
 
    fs.defaultFS
    hdfs://gagcluster:8020
 

 
    io.file.buffer.size
    131072
 

 
 
    hadoop.tmp.dir
    /usr/hadoop/storage/hadoop/tmp
 

 
 
    hadoop.proxyuser.root.hosts
    *
 

 
 
    hadoop.proxyuser.root.groups
    *
 

 
 
    ha.zookeeper.quorum
    SA01:2181,SA02:2181,SA03:2181
 

#################################################

[2]
vi /usr/hadoop/hadoop/etc/hadoop/hdfs-site.xml
################################################





  dfs.hosts.exclude
  /usr/hadoop/hadoop/etc/hadoop/exclude

 
    dfs.block.size
    67108864
 

 
    dfs.nameservices
    gagcluster
 

 
    dfs.ha.namenodes.gagcluster
    nn1,nn2
 

 
    dfs.namenode.rpc-address.gagcluster.nn1
    SA01:8020
 

 
    dfs.namenode.http-address.gagcluster.nn1
    SA01:50070
 

 
    dfs.namenode.rpc-address.gagcluster.nn2
    SA02:8020
 

 
    dfs.namenode.http-address.gagcluster.nn2
    SA02:50070
 

 
    dfs.namenode.shared.edits.dir
    qjournal://SA01:8485;SA02:8485;SA03:8485/gagcluster
 

 
    dfs.client.failover.proxy.provider.gagcluster
    org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
 

 
    dfs.ha.fencing.methods
    sshfence
 

 
    dfs.ha.fencing.ssh.private-key-files
    /root/.ssh/id_rsa
 

 
    dfs.journalnode.edits.dir
    /usr/hadoop/storage/hadoop/journal
 

 
    dfs.ha.automatic-failover.enabled
    true
 

 
    dfs.namenode.name.dir   
    /usr/hadoop/storage/hadoop/name
 

 
    dfs.datanode.data.dir
    file:/usr/hadoop/storage/hadoop/data
 

 
    dfs.replication
    2
 

 
    dfs.webhdfs.enabled
    true
 

 
    dfs.journalnode.http-address
    0.0.0.0:8480
 

 
    dfs.journalnode.rpc-address
    0.0.0.0:8485
 

 
    ha.zookeeper.quorum
    SA01:2181,SA02:2181,SA03:2181
 

#################################################
[3]
cp /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml.template /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml
vi /usr/hadoop/hadoop/etc/hadoop/mapred-site.xml
#################################################




 
    mapreduce.framework.name
    yarn
 

 
    mapreduce.jobhistory.address
    0.0.0.0:10020
 

 
    mapreduce.jobhistory.webapp.address
    0.0.0.0:19888
 

###################################################

[4]
vi /usr/hadoop/hadoop/etc/hadoop/yarn-site.xml
###################################################


 
 
     yarn.log-aggregation-enable
     true
 

 
 
     yarn.log-aggregation.retain-seconds
     259200
 

 
   
     yarn.resourcemanager.connect.retry-interval.ms 
     2000 
 

 
   
     yarn.resourcemanager.ha.enabled 
     true 
   

 
    yarn.resourcemanager.ha.rm-ids
    rm1,rm2
 

 
    ha.zookeeper.quorum
    SA01:2181,SA02:2181,SA03:2181 
 
  
 
   
     yarn.resourcemanager.ha.automatic-failover.enabled 
     true 
   

 
    yarn.resourcemanager.hostname.rm1
    SA01
 
                     
 
     yarn.resourcemanager.hostname.rm2
     SA02
 

 
   
    yarn.resourcemanager.ha.id 
    rm1
   

 
 
    yarn.resourcemanager.recovery.enabled 
    true 
 

 
   
    yarn.resourcemanager.zk-state-store.address 
    SA01:2181,SA02:2181,SA03:2181
   

   
    yarn.resourcemanager.store.class 
    org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore 
   

 
    yarn.resourcemanager.zk-address
    SA01:2181,SA02:2181,SA03:2181
 

   
    yarn.resourcemanager.cluster-id 
    gagcluster-yarn 
   

 
   
    yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms 
    5000 
   

 
   
    yarn.resourcemanager.address.rm1 
    SA01:8132 
   

   
    yarn.resourcemanager.scheduler.address.rm1 
    SA01:8130 
   

   
    yarn.resourcemanager.webapp.address.rm1 
    SA01:8188 
   

 
    yarn.resourcemanager.resource-tracker.address.rm1 
    SA01:8131 
   

   
    yarn.resourcemanager.admin.address.rm1 
    SA01:8033 
   

   
    yarn.resourcemanager.ha.admin.address.rm1 
    SA01:23142 
   

 
   
    yarn.resourcemanager.address.rm2 
    SA02:8132 
   

   
    yarn.resourcemanager.scheduler.address.rm2 
    SA02:8130 
   

   
    yarn.resourcemanager.webapp.address.rm2 
    SA02:8188 
   

 
    yarn.resourcemanager.resource-tracker.address.rm2 
    SA02:8131 
   

   
    yarn.resourcemanager.admin.address.rm2 
    SA02:8033 
   

   
    yarn.resourcemanager.ha.admin.address.rm2 
    SA02:23142 
   

   
    yarn.nodemanager.aux-services 
    mapreduce_shuffle 
   

   
    yarn.nodemanager.aux-services.mapreduce.shuffle.class 
    org.apache.hadoop.mapred.ShuffleHandler 
   

   
    yarn.nodemanager.local-dirs 
    /usr/hadoop/storage/yarn/local 
   

   
    yarn.nodemanager.log-dirs 
    /usr/hadoop/storage/yarn/logs 
   

   
    mapreduce.shuffle.port 
    23080 
   

 
   
    yarn.client.failover-proxy-provider 
    org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider 
   

 
      yarn.resourcemanager.ha.automatic-failover.zk-base-path
      /yarn-leader-election
 

#######################################################

配置DataNode节点
vi /usr/hadoop/hadoop/etc/hadoop/slaves
####################################################### 
slave141
slave142
slave143
Slave144
Slave145

vi /usr/hadoop/hadoop/etc/hadoop/*.env.sh（三个env.sh文件）
#######################################################
export JAVA_HOME=/usr/hadoop/jdk

创建exclude文件，用于以后下线hadoop节点
touch /usr/hadoop/hadoop/etc/hadoop/exclude

同步hadoop工程到hadoop002~005机器上面
for ip in `seq 140 145`;do scp -r /usr/hadoop/hadoop slave$ip:/usr/hadoop/;done

修改nn2配置文件yarn-site.xml
#####################################################
 
    yarn.resourcemanager.ha.id 
    rm2
 
#####################################################