Shell脚本之——Hadoop3单机版安装

目录

1.解压

2.文件重命名

3.配置环境变量

4.hadoop-env.sh

5.core-site.xml

6. hdfs-site.xml

7. mapred-site.xml

8.yarn-site.xml

9.完整脚本代码(注意修改主机名)

10.赋予执行权限

11.执行脚本

12.免密登录

13.时间同步

14.重启环境变量

15.初始化

16.启动服务

17.jps查询节点

18.网页验证

---

1.解压

tar -zxf /opt/install/hadoop-3.1.3.tar.gz -C /opt/soft/

2.文件重命名

mv /opt/soft/hadoop-3.1.3 /opt/soft/hadoop313

3.配置环境变量

  sed -i '73a\export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop' /etc/profile
  sed -i '73a\export HADOOP_LIBEXEC_DIR=$HADOOP_HOME/libexec' /etc/profile
  sed -i '73a\export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native' /etc/profile
  sed -i '73a\export HADOOP_INSTALL=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_YARN_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_HDFS_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_COMMON_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_MAPRED_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export YARN_NODEMANAGER_USER=root' /etc/profile
  sed -i '73a\export YARN_RESOURCEMANAGER_USER=root' /etc/profile
  sed -i '73a\export HDFS_ZKFC_USER=root' /etc/profile
  sed -i '73a\export HDFS_JOURNALNODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_SECONDARYNAMENODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_DATANODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_NAMENODE_USER=root' /etc/profile
  sed -i '73a\export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/lib' /etc/profile
  sed -i '73a\export HADOOP_HOME=/opt/soft/hadoop313' /etc/profile
  sed -i '73a\#HADOOP_HOME' /etc/profile

4.hadoop-env.sh

#hadoop-env.sh
  sed -i '53a\export JAVA_HOME=/opt/soft/jdk180'  /opt/soft/hadoop313/etc/hadoop/hadoop-env.sh

5.core-site.xml

#core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.proxyuser.root.groups' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.proxyuser.root.hosts' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    读写队列缓存：128k' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    131072' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    io.file.buffer.size' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    root' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.http.staticuser.user' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    namenode上本地的hadoop临时文件夹' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.tmp.dir' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hdfs://cp145:9000' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    fs.defaultFS' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml

6. hdfs-site.xml

#hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    关闭权限验证' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    false' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.permissions.enabled' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    datanode上数据块的物理存储位置目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data/dfs/data' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.datanode.data.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    namenode上存储hdfsq名字空间元数据的目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data/dfs/name' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.namenode.name.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    hadoop中每一个block文件的备份数量' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    1' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.replication' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml

7. mapred-site.xml

#mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.reduce.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.map.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    cp145:19888' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.jobhistory.webapp.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    cp145:10020' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.jobhistory.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    /opt/soft/hadoop313/etc/hadoop:/opt/soft/hadoop313/share/hadoop/common/lib/*:/opt/soft/hadoop313/share/hadoop/common/*:/opt/soft/hadoop313/share/hadoop/hdfs/*:/opt/soft/hadoop313/share/hadoop/hdfs/lib/*:/opt/soft/hadoop313/share/hadoop/mapreduce/*:/opt/soft/hadoop313/share/hadoop/mapreduce/lib/*:/opt/soft/hadoop313/share/hadoop/yarn/*:/opt/soft/hadoop313/share/hadoop/yarn/lib/*' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\   mapreduce.application.classpath' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    true' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    job执行框架：local,classic or yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.framework.name' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml

8.yarn-site.xml

  #yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    /opt/soft/hadoop313/yarndata/log' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.log-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    /opt/soft/hadoop313/yarndata/yarn' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.local-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    mapreduce_shuffle' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.aux-services' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8042' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.webapp.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8050' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8040' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.localizer.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.resourcemanager.scheduler.class' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    20000' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.resourcemanager.connect.retry-interval.ms' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml

9.完整脚本代码(注意修改主机名)

#! /bin/bash
echo 'auto install hadoop begining....'

#global var
hadoop=true

if [ "$hadoop" = true ];then
  echo 'hadoop install set true'
  echo 'setup hadoop 3'
  tar -zxf /opt/install/hadoop-3.1.3.tar.gz -C /opt/soft/
  mv /opt/soft/hadoop-3.1.3 /opt/soft/hadoop313
  sed -i '73a\export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop' /etc/profile
  sed -i '73a\export HADOOP_LIBEXEC_DIR=$HADOOP_HOME/libexec' /etc/profile
  sed -i '73a\export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native' /etc/profile
  sed -i '73a\export HADOOP_INSTALL=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_YARN_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_HDFS_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_COMMON_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export HADOOP_MAPRED_HOME=$HADOOP_HOME' /etc/profile
  sed -i '73a\export YARN_NODEMANAGER_USER=root' /etc/profile
  sed -i '73a\export YARN_RESOURCEMANAGER_USER=root' /etc/profile
  sed -i '73a\export HDFS_ZKFC_USER=root' /etc/profile
  sed -i '73a\export HDFS_JOURNALNODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_SECONDARYNAMENODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_DATANODE_USER=root' /etc/profile
  sed -i '73a\export HDFS_NAMENODE_USER=root' /etc/profile
  sed -i '73a\export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/lib' /etc/profile
  sed -i '73a\export HADOOP_HOME=/opt/soft/hadoop313' /etc/profile
  sed -i '73a\#HADOOP_HOME' /etc/profile
  
  #hadoop-env.sh
  sed -i '53a\export JAVA_HOME=/opt/soft/jdk180'  /opt/soft/hadoop313/etc/hadoop/hadoop-env.sh

  #core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.proxyuser.root.groups' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.proxyuser.root.hosts' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    读写队列缓存：128k' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    131072' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    io.file.buffer.size' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    root' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.http.staticuser.user' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    namenode上本地的hadoop临时文件夹' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hadoop.tmp.dir' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    hdfs://cp145:9000' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\    fs.defaultFS' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml
  
  #hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    关闭权限验证' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    false' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.permissions.enabled' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    datanode上数据块的物理存储位置目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data/dfs/data' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.datanode.data.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    namenode上存储hdfsq名字空间元数据的目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    /opt/soft/hadoop313/data/dfs/name' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.namenode.name.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    hadoop中每一个block文件的备份数量' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    1' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\    dfs.replication' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml
  
  #mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.reduce.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.map.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    cp145:19888' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.jobhistory.webapp.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    cp145:10020' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.jobhistory.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    /opt/soft/hadoop313/etc/hadoop:/opt/soft/hadoop313/share/hadoop/common/lib/*:/opt/soft/hadoop313/share/hadoop/common/*:/opt/soft/hadoop313/share/hadoop/hdfs/*:/opt/soft/hadoop313/share/hadoop/hdfs/lib/*:/opt/soft/hadoop313/share/hadoop/mapreduce/*:/opt/soft/hadoop313/share/hadoop/mapreduce/lib/*:/opt/soft/hadoop313/share/hadoop/yarn/*:/opt/soft/hadoop313/share/hadoop/yarn/lib/*' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\   mapreduce.application.classpath' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    true' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    job执行框架：local,classic or yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\    mapreduce.framework.name' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  sed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml
  
  #yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    /opt/soft/hadoop313/yarndata/log' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.log-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    /opt/soft/hadoop313/yarndata/yarn' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.local-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    mapreduce_shuffle' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.aux-services' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8042' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.webapp.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8050' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    cp145:8040' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.nodemanager.localizer.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.resourcemanager.scheduler.class' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    20000' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\    yarn.resourcemanager.connect.retry-interval.ms' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml
  sed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xml

  echo 'setup hadoop 3 success!!!'
fi

10.赋予执行权限

[root@lxm148 shell]# chmod 777 ./hadoopinstall.sh 
[root@lxm148 shell]# ll
total 16
-rwxrwxrwx. 1 root root 11438 Feb 17 16:27 hadoopinstall.sh

11.执行脚本

[root@lxm148 shell]# ./hadoopinstall.sh 
auto install hadoop begining....
hadoop install set true
setup hadoop 3
setup hadoop 3 success!!!

12.免密登录

Linux设置ssh免密登录

[root@lxm148 ~]# ssh lxm148
The authenticity of host 'lxm148 (fe80::3c58:ca96:14f8:7eb1%ens33)' cahed.
ECDSA key fingerprint is SHA256:W1WQtG0ruqDt0/o/WhDHFC/7DMdlIRNfdtrLzu
ECDSA key fingerprint is MD5:aa:50:2e:f5:9b:65:f1:4c:ab:06:d9:60:d2:e1
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lxm148,fe80::3c58:ca96:14f8:7eb1%ens33' (Est of known hosts.
root@lxm148's password: 
Last login: Fri Feb 17 16:21:54 2023 from 192.168.180.1


[root@lxm148 ~]# cd .ssh


[root@lxm148 .ssh]# ssh-keygen -t rsa

Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FntVWtMdqt+nIofnOgNVNMV15PJ3xzjc/2N4hVDG9Hs root@lxm148
The key's randomart image is:
+---[RSA 2048]----+
|           .o=*=*|
|            .=*++|
|        .  .o+. o|
|         o..o. *.|
|        S... .+oE|
|       ...  . o.B|
|         . . ...+|
|          = +. =o|
|          .O..+ o|
+----[SHA256]-----+

[root@lxm148 .ssh]# ll
total 12
-rw-------. 1 root root 1679 Feb 17 16:33 id_rsa
-rw-r--r--. 1 root root  393 Feb 17 16:33 id_rsa.pub
-rw-r--r--. 1 root root  200 Feb 17 16:30 known_hosts


[root@lxm148 .ssh]# ssh-copy-id lxm148
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
root@lxm148's password: 

Number of key(s) added: 1

Now try logging into the machine, with:   "ssh 'lxm148'"
and check to make sure that only the key(s) you wanted were added.

[root@lxm148 .ssh]# ssh lxm148
Last login: Fri Feb 17 16:30:48 2023 from fe80::3c58:ca96:14f8:7eb1%ens33
[root@lxm148 ~]# 

13.时间同步

《Linux中CentOS7时间与网络时间orWindows同步的方法》

14.重启环境变量

[root@lxm148 opt]# source /etc/profile

15.初始化

[root@lxm148 yarndata]# hdfs namenode -format

16.启动服务

[root@lxm148 yarndata]# start-all.sh

17.jps查询节点

18.网页验证