Hadoop分布式环境搭建

1. 安装配置虚拟机

VirtualMachine	IP	hostname
Hadoop_master	172.16.61.130	hadoopmaster
Hadoop_slaver01	172.16.61.131	hadoopslaver01
Hadoop_slaver02	172.16.61.132	hadoopslaver02

1. OS :CentOS-7-x86_64-DVD-1611.iso
2. 流程为先安装配置一个虚拟机，然后根据配置好的虚拟机克隆两个新的虚拟机，再修改两个新的虚拟机的hostname
3. 安装SSHD
   $yum install openssh-server

2. 安装hadoop

$ yum -y install wget
$ wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.8.0/hadoop-2.8.0.tar.gz
$ tar -zxvf hadoop-2.8.0.tar.gz
$ mv hadoop-2.8.0 hadoop
$ groupadd hadoop
$ useradd -g hadoop -m hadoop
$ passwd hadoop
$ id
$ chown -R hadoop:hadoop /usr/local/hadoop
$ yum -y install tree
$ tree hadoop -L 1
    hadoop
    ├── bin
    ├── etc
    ├── include
    ├── lib
    ├── libexec
    ├── LICENSE.txt
    ├── NOTICE.txt
    ├── README.txt
    ├── sbin
    └── share
$ mkdir -p /usr/local/hadoop/hdfs/name
$ mkdir -p /usr/local/hadoop/hdfs/data
$ chown -R hadoop:hadoop /usr/local/hadoop/hdfs
$ scp /Users/Tuple/Downloads/jdk-8u131-linux-x64.tar.gz [email protected]:/var/local
    
$ tar -zxvf jdk-8u131-linux-x64.tar.gz
$ mv jdk1.8.0_131 java
$ vi /etc/profile
    
    export JAVA_HOME=/var/local/java/
    export JRE_HOME=/var/local/java/jre
    export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH
    export PATH=$JAVA_HOME/bin:$PATH
        
$ source /etc/profile
$ java -version
$ javac
$ echo $JAVA_HOME

3. 配置Hadoop_master

1. /etc/profile
$ vi /etc/profile
export HADOOP_HOME=/usr/local/hadoop
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export HADOOP_MAPARED_HOME=${HADOOP_HOME}
export HADOOP_COMMON_HOME=${HADOOP_HOME}
export HADOOP_HDFS_HOME=${HADOOP_HOME}
export HADOOP_YARN_HOME=${HADOOP_HOME}
export YARN_HOME=${HADOOP_HOME}
export YARN_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export HDFS_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native/:$LD_LIBRARY_PATH
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
$ source /etc/profile
$ echo $HADOOP_HOME
    /usr/local/hadoop
    
2. slaves
$ vi $HADOOP_HOME/etc/hadoop/slaves
172.16.61.131
172.16.61.132

3. hadoop-env.sh
$ vi $HADOOP_HOME/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/var/local/java
$ source $HADOOP_HOME/etc/hadoop/hadoop-env.sh

4. core-site.xml
$ vi $HADOOP_HOME/etc/hadoop/core-site.xml
 
   
    fs.default.name  
    hdfs://hadoopmaster:9000 
    
   
    hadoop.tmp.dir  
    /usr/local/hadoop/hadoop_tmp 
    
   
    hadoop.proxyuser.root.hosts  
    * 
    
   
    hadoop.proxyuser.root.groups  
    * 
    
   
    hadoop.proxyuser.sqoop2.hosts  
    * 
    
   
    hadoop.proxyuser.sqoop2.groups  
    * 
   


5. hdfs-site.xml
$ vi $HADOOP_HOME/etc/hadoop/hdfs-site.xml

  
    dfs.namenode.secondary.http-address
    hadoopmaster:9001
  
  
    dfs.replication
    1
  
  
    dfs.tmp.dir
    /usr/local/hadoop/hadoop_tmp
  
  
    dfs.namenode.name.dir
    file:/usr/local/hadoop/hdfs/name
  
  
    dfs.datanode.data.dir
    file:/usr/local/hadoop/hdfs/data
  
  
    dfs.permissions
    false
  
  
    dfs.webhdfs.enabled
    true
  
  
    dfs.datanode.max.xcievers
    4096
  


6. mapred-site.xml
$ vi $HADOOP_HOME/etc/hadoop/mapred-site.xml

  
    mapreduce.framework.name
    yarn
  
  
    mapreduce.application.classpath
    /usr/local/hadoop/etc/hadoop,/usr/local/hadoop/share/hadoop/common/*,/usr/local/hadoop/share/hadoop/common/lib/*,/usr/local/hadoop/share/hadoop/hdfs/*,/usr/local/hadoop/share/hadoop/hdfs/lib/*,/usr/local/hadoop/share/hadoop/mapreduce/*,/usr/local/hadoop/share/hadoop/mapreduce/lib/*,/usr/local/hadoop/share/hadoop/yarn/*,/usr/local/hadoop/share/hadoop/yarn/lib/*
  
  
    mapreduce.jobhistory.address
    hadoopmaster:10020
  
  
    mapreduce.jobhistory.webapp.address
    hadoopmaster:19888
  
  
    mapreduce.map.memory.mb
    1536
  
  
    mapreduce.map.java.opts
    -Xmx3072M
  
  
    mapreduce.reduce.memory.mb
    3072
  
  
    mapreduce.reduce.java.opts
    -Xmx6144M
  
  
    mapreduce.cluster.map.memory.mb
    -1
  
  
    mapreduce.cluster.reduce.memory.mb
    -1
  


7. yarn-site.xml
$ vi $HADOOP_HOME/etc/hadoop/yarn-site.xml
 
    
   
    yarn.resourcemanager.address  
    hadoopmaster:8032 
    
   
    yarn.resourcemanager.scheduler.address  
    hadoopmaster:8030 
    
   
    yarn.resourcemanager.resource-tracker.address  
    hadoopmaster:8031 
    
   
    yarn.resourcemanager.admin.address  
    hadoopmaster:8033 
    
   
    yarn.resourcemanager.webapp.address  
    hadoopmaster:8088 
    
   
    yarn.nodemanager.aux-services  
    mapreduce_shuffle 
    
   
    yarn.nodemanager.aux-services.mapreduce.shuffle.class  
    org.apache.hadoop.mapred.ShuffleHandler 
    
   
    mapreduce.application.classpath  
    /usr/local/hadoop/etc/hadoop,/usr/local/hadoop/share/hadoop/common/*,/usr/local/hadoop/share/hadoop/common/lib/*,/usr/local/hadoop/share/hadoop/hdfs/*,/usr/local/hadoop/share/hadoop/hdfs/lib/*,/usr/local/hadoop/share/hadoop/mapreduce/*,/usr/local/hadoop/share/hadoop/mapreduce/lib/*,/usr/local/hadoop/share/hadoop/yarn/*,/usr/local/hadoop/share/hadoop/yarn/lib/* 
    
   
    yarn.nodemanager.vmem-pmem-ratio  
    3 
   

4. 配置Hadoop_slaver01、Hadoop_slaver02

1. 关闭Hadoop_master
$ sudo shutdown now

2. 在VMwareFusion中使用Hadoop_master克隆Hadoop_slaver01、Hadoop_slaver02

3. 修改克隆虚拟机的hostname
$ hostnamectl --static set-hostname hadoopslaver01/hadoopslaver02

4. 修改Hadoop_slaver01、Hadoop_slaver02的hosts文件，添加如下行
172.16.61.130 hadoopmaster
172.16.61.131 hadoopslaver01
172.16.61.132 hadoopslaver02

5. 配置Hadoop主从节点无密码登录

$ vi/etc/ssh/sshd_config
$ service sshd restart

1. 生成密钥
$ ssh-keygen -t dsa

2. 导入到本机AuthorizedKeysFile
$ cd /home/hadoop/.ssh/
$ cat id_dsa.pub >> authorized_keys

3. 导入到远程主机AuthorizedKeysFile
$ cd /home/hadoop/.ssh/
$ cat /home/hadoop/.ssh/id_dsa.pub | ssh hadoop@hadoopmaster 'cat - >> /home/hadoop/.ssh/authorized_keys'

4. 文件权限修改
chmod 600 /home/hadoop/.ssh/authorized_keys

6. 启动hadoop

# 确保hadoop配置目录的owner为hadoop用户，不是root

1. 格式化namenode,只需执行一次
$ hadoop namenode -format
$ chown -R hadoop:hadoop /usr/local/hadoop/hadoop_tmp
$ chown -R hadoop:hadoop /usr/local/hadoop/logs

2. 关闭防火墙
$ systemctl stop firewalld.service  
$ systemctl disable firewalld.service

3. 启动dfs及yarn
$ $HADOOP_HOME/sbin/start-dfs.sh
$ $HADOOP_HOME/sbin/start-yarn.sh
-> 或者以下命令都启动
$ $HADOOP_HOME/sbin/start-all.sh

4. 启动history服务
$ $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver

5. 停止集群
$ $HADOOP_HOME/sbin/stop-dfs.sh
$ $HADOOP_HOME/sbin/stop-yarn.sh
$ $HADOOP_HOME/stop-all.sh

6. 查看进程启动情况
$ jps

7. WEB监控
http://172.16.61.130:50070/
http://172.16.61.130:8088/

8. 测试
$ cd  $HADOOP_HOME
$ $HADOOP_HOME/bin/hadoop fs -put #HADOOP_HOME/hadoop.txt /tmp/input
$ hdfs dfs -ls /tmp/input
$ hadoop fs -rm -r /tmp/input
$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.0.jar wordcount /tmp/input output
$ hadoop fs -ls output/
$ hadoop fs -cat output/part-r-00000

9. 查看集群状态
$ hdfs dfsadmin -report