高可用Hadoop集群搭建

配置

1. 配置hosts

在/etc/hosts中配置ip与机器名关系

192.168.0.1   lc2
192.168.0.2   lc3
192.168.0.3   lc4

2. 配置无秘钥登录

生成秘钥

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

拷贝
ssh-copy-id -i ~/.ssh/id_rsa.pub user@st1

3. 配置java环境

省略

4. Zookeeper集群搭建

集群搭建

5. Hadoop配置

core.xml


        
        
            fs.defaultFS
            hdfs://ns
        
    
        
            hadoop.tmp.dir
            /data/hadoop/tmp
        
        
        
        
            ha.zookeeper.quorum
            lc2:2181,lc3:2181,lc4:2181

hdfs-site.xml


    
    
        dfs.nameservices
        ns
    
    
    
        dfs.ha.namenodes.ns
        nn1,nn2
    
    
    
        dfs.namenode.rpc-address.ns.nn1
        lc2:9000
    
    
    
        dfs.namenode.http-address.ns.nn1
        lc2:50070
    
    
    
        dfs.namenode.rpc-address.ns.nn2
        lc3:9000
    
    
    
        dfs.namenode.http-address.ns.nn2
        lc3:50070
    
    
    
        dfs.namenode.shared.edits.dir
        qjournal://lc2:8485;lc3:8485;lc4:8485/ns
    
    
    
        dfs.journalnode.edits.dir
        /data/hadoop/journal
    
    
    
        dfs.ha.automatic-failover.enabled
        true
    
    
    
        dfs.client.failover.proxy.provider.ns
        org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
    
    
    
        dfs.ha.fencing.methods
        sshfence
    
    
    
        dfs.ha.fencing.ssh.private-key-files
        /home/magneto/.ssh/id_rsa
    

    
    
        dfs.webhdfs.enabled
        true
    
    
        dfs.permissions
        false
    
    
        dfs.replication
        3
    
    
        dfs.namenode.name.dir
        file:///data/hadoop/name
    
    
        dfs.datanode.data.dir
        file:///data/hadoop/data

yarn-site.xml


    
        yarn.resourcemanager.ha.enabled
        true
    
    
        yarn.resourcemanager.cluster-id
        rs
    
    
        yarn.resourcemanager.ha.rm-ids
        rm1,rm2
    
    
        yarn.resourcemanager.hostname.rm1
        lc3
    
    
        yarn.resourcemanager.hostname.rm2
        lc4
    
    
        yarn.resourcemanager.webapp.address.rm1
        lc3:8088
    
    
        yarn.resourcemanager.webapp.address.rm2
        lc4:8088
    
    
        yarn.resourcemanager.zk-address
        lc2:2181,lc3:2181,lc4:2181
    
    
        yarn.nodemanager.vmem-pmem-ratio
        1.5
    

    
        yarn.nodemanager.resource.memory-mb
        50000
    

    
        yarn.nodemanager.resource.cpu-vcores
        80
    

    
        yarn.resourcemanager.scheduler.class
        org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
    

    
        yarn.scheduler.minimum-allocation-mb
        2048
    
    
        yarn.scheduler.maximum-allocation-mb
        32768
    
    
        yarn.scheduler.maximum-allocation-vcores
        24
    
    
        yarn.scheduler.minimum-allocation-vcores
        1
    
    
        yarn.acl.enable
        true
        使用使用ACL，默认是false
    

    
        yarn.nodemanager.aux-services
        mapreduce_shuffle

mapred-site.xml


    
        mapreduce.framework.name
        yarn
    
    
        mapreduce.cluster.acls.enabled
        true

slaves

lc2
lc3
lc4

6. 配置Hadoop环境变量

export HADOOP_HOME=/usr/local/hadoop
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export YARN_HOME=$HADOOP_HOME
export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

接下来拷贝hadoop目录到所有机器上。

启动

1.启动zookeeper集群

在所有配置了zookeeper机器上执行sh zkServer.sh start

2.启动journalno

在所有journalnode上执行sbin/hadoop-daemons.sh start journalnode

4.格式化zkfc,让在zookeeper中生成ha节点

在lc2上执行如下命令，完成格式化
hdfs zkfc –formatZK
格式化hdfs，执行hadoop namenode –format

5.启动namenode

首先在lc2上启动active节点，在lc2上执行如下命令sbin/hadoop-daemon.sh start namenode
在lc3上同步namenode的数据，执行hdfs namenode –bootstrapStandby
启动lc3上的namenode作为standby，执行sbin/hadoop-daemon.sh start namenode

6.启动datanode

执行sbin/hadoop-daemons.sh start datanode

7.启动ZKFC

在lc2上执行如下命令，完成ZKFC的启动
执行sbin/hadoop-daemons.sh start zkfc

8.启动yarn

执行start-yarn.sh
yarn-daemons.sh start resourcemanager