03.hadoop完全分布式系统环境搭建

一. 机器规划

192.168.28.22：namenode
192.168.28.23：resourcemanager
192.168.28.24：histroyservery与secondarynamenode

二.MAC地址修改

# 查看MAC地址
more /etc/udev/rules.d/70-persistent-net.rules
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:0c:29:87:62:86", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"
# 与网络配置中的MAC地址保持一致
more /etc/sysconfig/network-scripts/ifcfg-eth0 

三.ssh免密码登录

cd ~    #切换用户主目录
rm -rf .ssh  #删除.ssh文件
ssh-keygen -t rsa  #使用rsa加密
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
66:cc:1a:e5:18:e5:5d:ea:f7:7c:a6:82:6e:e1:32:da [email protected]
The key's randomart image is:
+--[ RSA 2048]----+
|        .   .    |
|       o . o     |
|      . o o      |
|       B .       |
|      o S . .    |
|       =  .. o   |
|      .  . o  o o|
|       .o + .  + |
|      ..E=.  ..  |
+-----------------+

ssh-copy-id bigguider22.com  #连接远程主机
The authenticity of host 'bigguider22.com (192.168.28.22)' can't be established.
RSA key fingerprint is a4:b8:5c:8c:a8:03:19:bb:07:9a:3c:95:c1:2f:32:0e.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'bigguider22.com,192.168.28.22' (RSA) to the list of known hosts.
[email protected]'s password: 
Now try logging into the machine, with "ssh 'bigguider22.com'", and check in:

  .ssh/authorized_keys

to make sure we haven't added extra keys that you weren't expecting.


ssh bigguider22.com #测试

四.配置文件修改

# core-site.xml


    
    fs.defaultFS 
    hdfs://bigguider22.com:8020


    
    hadoop.tmp.dir
    /root/modules/hadoop-2.5.0-cdh5.3.6/data/tmp



# hdfs-site.xml


   
   dfs.replication
   1


    dfs.namenode.secondary.http-address
    bigguider24.com:50090


    dfs.permissions.enabled
    false



# yarn-site.xml

 
   yarn.nodemanager.aux-services
   mapreduce_shuffle

     
     

   yarn.resourcemanager.hostname
   bigguider23.com




# mared-site.xml



  mapreduce.framework.name
  yarn


  mapreduce.jobhistory.address
  bigguider24.com:10020



mapreduce.jobhistory.webapp.address
   bigguider24.com:19888

五.配置文件分发

scp -r modules/hadoop-2.5.0-cdh5.3.6/ bigguider23.com:/root/modules
scp -r modules/hadoop-2.5.0-cdh5.3.6/ bigguider24.com:/root/modules

六. 启动服务

# bigguider22.com
hdfs namenode -format
hadoop-daemon.sh start namenode
hadoop-daemon.sh start datanode
yarn-daemon.sh start nodemanager

# bigguider23.com
hadoop-daemon.sh start datanode
yarn-daemon.sh start resourcemanager
yarn-daemon.sh start nodemanager

# bigguider24.com
hadoop-daemon.sh start datanode
yarn-daemon.sh start nodemanager
hadoop-daemon.sh start secondarynamenode
mr-jobhistory-daemon.sh start historyserver

六.时间同步

三台服务器:(修改系统时间 date -s 2017/3/25 )
bigguider22.com 192.168.28.22
bigguider23.com 192.168.28.23
bigguider24.com 192.168.28.24
已bigguider22.com这台机器的时间为标准同步另外两台机器。

1. 配置bigguider22.com机器

#1. 服务设置（ntpd未安装，请自行安装）
service ntpd status
service ntpd start
chkconfig ntpd on
#2. 编辑文件
vim /etc/ntp.conf
driftfile /var/lib/ntp/drift
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict -6 ::1
# 1. 代开此处设置对内网主机提供NTP服务
restrict 192.168.28.0 mask 255.255.255.0 nomodify notrap
# 2. 注释掉这些centos默认的对时服务
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst

# 3. 添加这两个，外部时间服务器不可用时，设置以本地时间作为时间服务
server 127.127.1.0 # local clock
fudge 127.127.1.0 stratum 10 
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys

#3. 重启服务
service ntpd restart

2. 配置其他两台机器

# 同步时间
/usr/sbin/ntpdate  bigguider22.com
# 定时执行时间同步
corntab -a  #表示每隔10分钟同步一次时间
0-59/10 * * * * /usr/sbin/ntpdate  bigguider22.com