Hadoop2.6 全分布式安装,配置了联邦

Hadoop全分布式安装2.6   共4台机器
hadoop01  namenode             zookeeper                  zkfc(zookeeper failover controller)
hadoop02  namenode  datanode   zookeeper   journalnode    zkfc
hadoop03            datanode   zookeeper   journalnode
hadoop04            datanode               journalnode
修改主机名称 永久生效  重启
vi /etc/sysconfig/network
HOSTNAME=hadoop01
HOSTNAME=hadoop02
HOSTNAME=hadoop03
HOSTNAME=hadoop04
临时生效
hostname  hadoop01
设置主机名称
vi /etc/hosts
192.168.192.133  hadoop01
192.168.192.134  hadoop02
192.168.192.135  hadoop03
192.168.192.136  hadoop04
scp /etc/hosts root@hadoop02:/etc
scp /etc/hosts root@hadoop03:/etc
scp /etc/hosts root@hadoop04:/etc
ssh登陆其他机器
ssh 192.168.192.133  -l root 
输入密码
123456
配置ssh无密码登陆
cd ~
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa           生成公钥、私钥对
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys          导入公钥
ssh localhost    测试
登录其它机器 ssh hadoop01  -l root
cd ~
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa           生成公钥、私钥对
然后用scp命令，把公钥文件发放给hadoop01
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_02.pub
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_03.pub
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_04.pub
登陆hadoop01   导入公钥
cat ~/.ssh/id_dsa_02.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa_03.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa_04.pub >> ~/.ssh/authorized_keys
将hadoop01上的“最全”公钥，复制到其它机器
scp ~/.ssh/authorized_keys root@hadoop02:/root/.ssh/authorized_keys
scp ~/.ssh/authorized_keys root@hadoop03:/root/.ssh/authorized_keys
scp ~/.ssh/authorized_keys root@hadoop04:/root/.ssh/authorized_keys
安装JDK 略过
上传并解压hadoop2.6
tar -zxvf hadoop-2.6.0-x64.tar.gz -C /opt/modules/
cd  /opt/modules/hadoop-2.6.0/etc/hadoop
vi hadoop-env.sh
export JAVA_HOME=/opt/modules/jdk1.7
vi yarn-env.sh
export JAVA_HOME=/opt/modules/jdk1.7
vi mapred-env.sh  
export JAVA_HOME=/opt/modules/jdk1.7
vi core-site.xml  tmp目录如不存在，则先mkdir手动创建

   
    hadoop.tmp.dir
    /opt/hadoop/tmp
   
   
    fs.defaultFS
    hdfs://mycluster
   

   ha.zookeeper.quorum
   hadoop01:2181,hadoop02:2181,hadoop03:2181
 

rm -rf /opt/hadoop/
mkdir /opt/hadoop  
mkdir /opt/hadoop/tmp
mkdir /opt/hadoop/journal
vi hdfs-site.xml  dfs.namenode.name.dir和dfs.namenode.edits.dir所设置的目录都应该手工建立好
配置联邦两个namenode

  dfs.nameservices
  mycluster


  dfs.ha.namenodes.mycluster
  nn1,nn2


  dfs.namenode.rpc-address.mycluster.nn1
  hadoop01:8020


  dfs.namenode.rpc-address.mycluster.nn2
  hadoop02:8020


  dfs.namenode.http-address.mycluster.nn1
  hadoop01:50070


  dfs.namenode.http-address.mycluster.nn2
  hadoop02:50070


  dfs.namenode.shared.edits.dir
  qjournal://hadoop02:8485;hadoop03:8485;hadoop04:8485/mycluster


  dfs.client.failover.proxy.provider.mycluster
  org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider


  dfs.ha.fencing.methods
  sshfence


  dfs.ha.fencing.ssh.private-key-files
  /root/.ssh/id_dsa


  dfs.journalnode.edits.dir
  /opt/hadoop/journal


   dfs.ha.automatic-failover.enabled
   true
  

dfs.replication
2





cp /opt/modules/hadoop-2.6.0/etc/hadoop/mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

mapreduce.framework.name
yarn



vi yarn-site.xml

yarn.nodemanager.aux-services
mapreduce_shuffle

vi slaves
hadoop02
hadoop03
hadoop04
安装zookeeper
tar -zxvf zookeeper-3.4.6.tar.gz -C /opt/modules/
cd /opt/modules/zookeeper-3.4.6/conf
cp zoo_sample.cfg zoo.cfg
vi zoo.cfg
mkdir /opt/zookeeper
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zookeeper
server.1=hadoop01:2888:3888
server.2=hadoop02:2888:3888
server.3=hadoop03:2888:3888
cd /opt/zookeeper       在hadoop01操作
vi myid
1
ssh hadoop02 -l root
scp -r /opt/zookeeper root@hadoop02:/opt/
vi myid
2
ssh hadoop03 -l root
scp -r /opt/zookeeper root@hadoop03:/opt/
vi myid
3
scp -r /opt/zookeeper root@hadoop03:/opt/
scp -r /opt/modules/zookeeper-3.4.6 root@hadoop02:/opt/modules
scp -r /opt/modules/zookeeper-3.4.6 root@hadoop03:/opt/modules
vi /etc/profile
export ZK_HOME=/opt/modules/zookeeper-3.4.6
export PATH=$PATH:$ZK_HOME/bin
source /etc/profile
scp /etc/profile root@hadoop02:/etc/
scp /etc/profile root@hadoop03:/etc/
启动zookeeper 01-02-03
zkServer.sh  start
停止ZooKeeper服务
zkServer.sh stop 
jps   查看是否启动
2516 QuorumPeerMain
拷贝hadoop
scp -r /opt/modules/hadoop-2.6.0 root@hadoop02:/opt/modules
scp -r /opt/modules/hadoop-2.6.0 root@hadoop03:/opt/modules
scp -r /opt/modules/hadoop-2.6.0 root@hadoop04:/opt/modules
02-03-04分别创建目录
mkdir /opt/hadoop
mkdir /opt/hadoop/tmp
mkdir /opt/hadoop/journal
mkdir /opt/hadoop/dfs
mkdir /opt/hadoop/dfs/data
mkdir /opt/hadoop/dfs/name
启动Hadoop
cd /opt/modules/hadoop-2.6.0/sbin/
02-03-04启动journalnode
./hadoop-daemon.sh start journalnode
格式化 zkfc(zookeeper failover controller)
01
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs zkfc -formatZK
格式化namenode
01
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs namenode -format
启动一个namenode
cd /opt/modules/hadoop-2.6.0/sbin/
./hadoop-daemon.sh start namenode
在没有格式化的namenode上执行
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs namenode -bootstrapStandby
停止服务(zookeeper除外)
./stop-dfs.sh
全部启动
./start-dfs.sh
测试访问
hadoop01:50070
hadoop02:50070
启动yarn
./start-yarn.sh
查看日志前50行
tail -n50 /opt/modules/hadoop-2.6.0/logs/hadoop-root-  .log
配置hadoop环境变量
vi /etc/profile
##HADOOP
export HADOOP_HOME=/opt/modules/hadoop-2.6.0
export PATH=$PATH:/opt/modules/hadoop-2.6.0/sbin:/opt/modules/hadoop-2.6.0/bin
source /etc/profile
scp /etc/profile root@hadoop02:/etc/
scp /etc/profile root@hadoop03:/etc/
scp /etc/profile root@hadoop04:/etc/
拷贝hadoop配置
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop02:/opt/modules/hadoop-2.6.0
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop03:/opt/modules/hadoop-2.6.0
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop04:/opt/modules/hadoop-2.6.0
每次启动01-02-03
zkServer.sh  start
全部启动
./start-dfs.sh
启动yarn
./start-yarn.sh
停止服务(zookeeper除外)
./stop-dfs.sh
./stop-yarn.sh


批量关机小脚本

#!/bin/bash
ip_array=("hadoop02" "hadoop03" "hadoop04") 
for ip in ${ip_array[*]}  
do  
    ssh  root@$ip "shutdown -h now"  
done 
echo "shutdown02 03 04" 
ssh  root@hadoop01 "shutdown -h now"  


chmod +x ./test.sh