HDFS HA with QJM && ResourceManager HA配置

背景：
两台机器，ubuntu21(10.67.2.21)和ubuntu20(10.67.2.20)


master为10.67.2.21
slave为10.67.2.20


两台机器上都有zookeeper。


用HA后，期望21上的Namenode和ResourceManager是active的，而20上的是standby；遇到namenode/resourcemanager down掉的情况，可以自动切换到standby的节点上


配置如下：

1.core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://mycluster</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/local/hadoop/tmpdir</value>
        </property>
        <property>
                <name>fs.checkpoint.period</name>
                <value>300</value>
        </property>
        <property>
                <name>fs.checkpoint.dir</name>
                <value>/usr/local/hadoop/tmpdir/namesecondary</value>
        </property>
        <property>
                <name>dfs.journalnode.edits.dir</name>
                <value>/usr/local/hadoop/tmpdir/journal/data</value>
        </property>
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>10.67.2.21:2181,10.67.2.20:2181</value>
    </property>
</configuration>

2.hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>dfs.replication</name>
                <value>1</value>
        </property>
        <property>
                <name>dfs.httpd.address</name>
                <value>ubuntu21:50070</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/local/hadoop/tmpdir</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/usr/local/hadoop/tmpdir/dfs/data</value>
        </property>
        <property>
                <name>dfs.webhdfs.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>dfs.nameservices</name>
                <value>mycluster</value>
        </property>
        <property>
                <name>dfs.ha.namenodes.mycluster</name>
                <value>nn1,nn2</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.mycluster.nn1</name>
                <value>ubuntu21:8020</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.mycluster.nn2</name>
                <value>ubuntu20:8020</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.mycluster.nn1</name>
                <value>ubuntu21:50070</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.mycluster.nn2</name>
                <value>ubuntu20:50070</value>
        </property>
        <property>
                <name>dfs.namenode.shared.edits.dir</name>
                <value>qjournal://ubuntu21:8485;ubuntu20:8485/mycluster</value>
        </property>
        <property>
                <name>dfs.client.failover.proxy.provider.mycluster</name>
                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        <property>
                <name>dfs.ha.fencing.methods</name>
                <value>shell(/bin/true)</value>
        </property>
        <property>
                <name>dfs.ha.automatic-failover.enabled</name>
                <value>true</value>
        </property>
</configuration>

3.yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>ubuntu21</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
                <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>pijing-yarn-ha</value>
    </property>
    <property>
    <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>ubuntu21</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>ubuntu20</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address.rm1</name>
        <value>ubuntu21:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>ubuntu20:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>10.67.2.21:2181,10.67.2.20:2181</value>
    </property>
</configuration>

4.mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

5.在ubuntu21和ubuntu20上分别运行命令：
nohup $HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode >/dev/null 2>&1 &


启动Journal Nodes (因为format namenode时它需要连接JNs)


6.在ubuntu21上运行：
$HADOOP_HOME/bin/hdfs namenode -format
格式化namenode


7.因为我们配置的dfs.datanode.data.dir是/usr/local/hadoop/tmpdir/dfs/data
所以我们把整个dfs打包，然后解压缩到ubuntu20的/usr/local/hadoop/tmpdir下(即它们用相同的namenode metadata)


8.在ubuntu21上运行
$HADOOP_HOME/bin/hdfs zkfc -formatZK


可以看到ubuntu21和ubuntu20的/usr/local/hadoop/tmpdir/journal/data下都会生成mycluster文件夹


9.在ubuntu21上运行
$HADOOP_HOME/sbin/start-all.sh可以自动启动ZKFC daemon


此时JPS查看ubuntu21上会有
NameNode
DataNode
ResourceManager
NodeManager
DFSZKFailoverController
JournalNode


此时JPS查看ubuntu20上会有
DataNode
NodeManager
DFSZKFailoverController
JournalNode


10.在ubuntu20上运行
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
可以看到ubuntu20上也会多一个ResourceManager


11.进入$HADOOP_HOME/bin


hdfs haadmin -getServiceState nn1 可以看到是active状态
hdfs haadmin -getServiceState nn2 可以看到是standby状态
yarn rmadmin -getServiceState rm1 可以看到是active状态
yarn rmadmin -getServiceState rm2 可以看到是standby状态