CentOS7 Hadoop-2.7.4 分布式部署

下载Hadoop

wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.7.4/hadoop-2.7.4.tar.gz

官方地址：http://mirror.bit.edu.cn/apache/hadoop/common/

如果wget不存在，使用如下命令安装

yum install -y wget

系统配置

host（别名不能出现.号）

192.168.1.211 hd211
192.168.1.212 hd212
192.168.1.214 hd214

免密码登录

cd /root/.ssh/
ssh-keygen -t rsa

一路回车

同步到其它服务器

ssh-copy-id hd211
ssh-copy-id hd212
ssh-copy-id hd214

安装、配置Hadoop

解压安装包

tar -zxvf hadoop-2.7.4.tar.gz

进入Hadoop配置文件目录

cd hadoop-2.7.4/

编辑hadoop-env.sh

vi etc/hadoop/hadoop-env.sh

export JAVA_HOME=/usr/lib/jvm/jre

export HADOOP_LOG_DIR=/data/hadoop/logs

编辑yarn-env.sh

vi etc/hadoop/yarn-env.sh

YARN_LOG_DIR="/data/hadoop/logs"

编辑core-site.xml

vi etc/hadoop/core-site.xml

内容

<configuration>
  <property>
    <name>fs.defaultFSname>
    <value>hdfs://hd212:9000value>
  property>
  <property>
    <name>io.file.buffer.sizename>
    <value>131072value>
  property>
  <property>
    <name>hadoop.tmp.dirname>
    <value>file:/data/hadoop/tmpvalue>
    <description>Abase for other temporary   directories.description>
  property>
configuration>

编辑hdfs-site.xml

vi etc/hadoop/hdfs-site.xml

内容

<configuration>
  <property>
    <name>dfs.permissions.enabledname>
    <value>falsevalue>
  property>
  <property>
    <name>dfs.webhdfs.enabledname>
    <value>truevalue>
  property>
  <property>
    <name>dfs.blocksizename>
    <value>64mvalue>
  property>
  <property>
    <name>dfs.nameservicesname>
    <value>hadoop-cluster-zgwvalue>
  property>
  
  <property>
    <name>dfs.replicationname>
    <value>3value>
  property>
  <property>
    <name>dfs.namenode.name.dirname>
    <value>/data/hadoop/hdfs/namevalue>
  property>
  <property>
    <name>dfs.namenode.checkpoint.dirname>
    <value>/data/hadoop/hdfs/checkpointvalue>
  property>
  <property>
    <name>dfs.namenode.checkpoint.edits.dirname>
    <value>/data/hadoop/hdfs/checkpointvalue>
  property>
  <property>
    <name>dfs.datanode.data.dirname>
    <value>/data/hadoop/hdfs/datavalue>
  property>
  <property>
    <name>dfs.namenode.secondary.http-addressname>
    <value>hd211:50090value>
  property>
configuration>

编辑mapred-site.xml

cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
vi etc/hadoop/mapred-site.xml

内容

<configuration>
  <property>
    <name>mapreduce.framework.namename>
    <value>yarnvalue>
  property>
  <property>
    <name>mapreduce.jobhistory.addressname>
    <value>hd212:10020value>
  property>
  <property>
    <name>mapreduce.jobhistory.webapp.addressname>
    <value>hd212:19888value>
  property>
configuration>

编辑yarn-site.xml

vi etc/hadoop/yarn-site.xml

内容

<configuration>
  <property>
    <name>yarn.nodemanager.aux-servicesname>
    <value>mapreduce_shufflevalue>
  property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.classname>
    <value>org.apache.hadoop.mapred.ShuffleHandlervalue>
  property>
  <property>
    <name>yarn.nodemanager.local-dirsname>
    <value>/data/hadoop/yarn/nodemanagervalue>
  property>
  <property>
    <name>yarn.resourcemanager.hostnamename>
    <value>hd212value>
  property>
  <property>
    <name>yarn.resourcemanager.addressname>
    <value>hd212:8032value>
  property>
  <property>
    <name>yarn.resourcemanager.scheduler.addressname>
    <value>hd212:8030value>
  property>
  <property>
    <name>yarn.resourcemanager.resource-tracker.addressname>
    <value>hd212:8031value>
  property>
  <property>
    <name>yarn.resourcemanager.admin.addressname>
    <value>hd212:8033value>
  property>
  <property>
    <name>yarn.resourcemanager.webapp.addressname>
    <value>hd212:8034value>
  property>
configuration>

编辑slaves

vi etc/hadoop/slaves

内容

hd211
hd212
hd214

环境变量

vi /etc/profile

#jdk
export JAVA_HOME=/usr/lib/jvm/jre
export PATH=$PATH:$JAVA_HOME/bin 

#hadoop  
export HADOOP_HOME=/usr/hadoop/hadoop-2.7.4  
export PATH=$PATH:$HADOOP_HOME/sbin  
export PATH=$PATH:$HADOOP_HOME/bin 

source /etc/profile

创建目录

mkdir /data/hadoop
mkdir /data/hadoop/hdfs
mkdir /data/hadoop/hdfs/data
mkdir /data/hadoop/hdfs/checkpoint
mkdir /data/hadoop/hdfs/name
mkdir /data/hadoop/yarn
mkdir /data/hadoop/yarn/nodemanager
mkdir /data/hadoop/tmp

防火墙配置

firewall-cmd --permanent --add-port=9000/tcp
firewall-cmd --permanent --add-port=50070/tcp
firewall-cmd --permanent --add-port=50090/tcp
firewall-cmd --permanent --add-port=10020/tcp
firewall-cmd --permanent --add-port=19888/tcp
firewall-cmd --permanent --add-port=8030/tcp
firewall-cmd --permanent --add-port=8031/tcp
firewall-cmd --permanent --add-port=8032/tcp
firewall-cmd --permanent --add-port=8033/tcp
firewall-cmd --permanent --add-port=8088/tcp
firewall-cmd --reload

格式华HDFS文件系统

hdfs namenode –format

启动hadoop

start-all.sh

浏览器访问

hd212:50070

其他

1.管理界面只能看到一个DataNode

2.SHUTDOWN_MSG: Shutting down NameNode at localhost/127.0.0.1
注意 -format 是否为中文符号

详细配置参考链接