-----------------------------
1.搭建手工切换HA(high availibility)的HDFS集群
-----------------------------
namenode:hadoop1和hadoop2
datanode:hadoop3、hadoop4、hadoop5、hadoop6
journalnode:hadoop1、hadoop2、hadoop3
export PATH=.:$JAVA_HOME/bin:$PATH
(3)source /etc/profile
<property>
<name>fs.defaultFS</name>
<value>hdfs://cluster1</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
1.搭建手工切换HA(high availibility)的HDFS集群
-----------------------------
namenode:hadoop1和hadoop2
datanode:hadoop3、hadoop4、hadoop5、hadoop6
journalnode:hadoop1、hadoop2、hadoop3
1.0 配置linux环境
1.0.1 确保虚拟机直接可以ping成功
1.0.1 确保虚拟机直接可以ping成功
1.0.2 设置hostname
vi /etc/sysconfig/network
vi /etc/sysconfig/network
1.0.3 配置/etc/hosts
vi /etc/hosts
vi /etc/hosts
1.0.4 关闭防火墙
service iptables stop
chkconfig iptables off
service iptables stop
chkconfig iptables off
1.0.5 实现ssh免密码登录
ssh-keygen -t rsa
ssh-copy-id -i hadoop2(被登录登录的机器)
1.0.6 jdk安装
(1)下载,并解压
(2)修改环境变量 vi /etc/profile 增加内容如下:
export JAVA_HOME=/usr/local/jdk
export PATH=.:$JAVA_HOME/bin:$PATH
(3)source /etc/profile
验证: java -version
1.1 配置文件(hadoop-env.sh、core-site.xml、hdfs-site.xml、slaves)
1.1.1 hadoop-env.sh
export JAVA_HOME=/usr/local/jdk1.7.0-45
export JAVA_HOME=/usr/local/jdk1.7.0-45
1.1.2 core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://cluster1</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
1.1.3 hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>cluster1</value>
</property>
<property>
<name>dfs.ha.namenodes.cluster1</name>
<value>hadoop1,hadoop2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cluster1.hadoop1</name>
<value>hadoop1:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.cluster1.hadoop1</name>
<value>hadoop1:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cluster1.hadoop2</name>
<value>hadoop2:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.cluster1.hadoop2</name>
<value>hadoop2:50070</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled.cluster1</name>
<value>false</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/cluster1</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/usr/local/hadoop/tmp/journal</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.cluster1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>cluster1</value>
</property>
<property>
<name>dfs.ha.namenodes.cluster1</name>
<value>hadoop1,hadoop2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cluster1.hadoop1</name>
<value>hadoop1:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.cluster1.hadoop1</name>
<value>hadoop1:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cluster1.hadoop2</name>
<value>hadoop2:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.cluster1.hadoop2</name>
<value>hadoop2:50070</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled.cluster1</name>
<value>false</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/cluster1</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/usr/local/hadoop/tmp/journal</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.cluster1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
1.1.4 slaves
hadoop3
hadoop4
hadoop5
hadoop6
hadoop3
hadoop4
hadoop5
hadoop6
1.1.5 把hadoop1上的hadoop文件夹复制到hadoop2、hadoop3、hadoop4、hadoop5、hadoop6节点
scp -r hadoop2.5 hadoop2:/home/hadoop/opt
1.2 启动journalnode集群
在hadoop1、hadoop2、hadoop3上分别执行hadoop/sbin/hadoop-daemon.sh start journalnode
在hadoop1、hadoop2、hadoop3上分别执行hadoop/sbin/hadoop-daemon.sh start journalnode
1.3 格式化namenode、启动namenode
在hadoop1上执行hadoop/bin/hdfs namenode -format
在hadoop1上分别执行hadoop/sbin/hadoop-daemon.sh start namenode
在hadoop2上执行hadoop/bin/hdfs namenode -bootstrapStandby
在hadoop2上分别执行hadoop/sbin/hadoop-daemon.sh start namenode
在hadoop1上执行hadoop/bin/hdfs haadmin -failover --forceactive hadoop2 hadoop1
在hadoop1上执行hadoop/bin/hdfs namenode -format
在hadoop1上分别执行hadoop/sbin/hadoop-daemon.sh start namenode
在hadoop2上执行hadoop/bin/hdfs namenode -bootstrapStandby
在hadoop2上分别执行hadoop/sbin/hadoop-daemon.sh start namenode
在hadoop1上执行hadoop/bin/hdfs haadmin -failover --forceactive hadoop2 hadoop1
1.4 启动datanode
在hadoop1上分别执行hadoop/sbin/hadoop-daemons.sh start datanode
在hadoop1上分别执行hadoop/sbin/hadoop-daemons.sh start datanode