docker搭建hadoop完全分布式

集群规划
1个NameNode节点
1个SecondaryNameNode节点
1个ResourceManager节点
1个JobHistory节点
2个Slave节点
1个Client节点

1. 环境准备

hadoop、jdk解压备用
 - docker
 - docker-compose
 - hadoop-3.2.3.tar.gz
 - jdk-11.0.15_linux-x64_bin.tar.gz

2. 基础镜像制作
   2.1 下载相关工具 如 ssh、vim等
   2.2 上传hadoop、jdk
   
   2.3 配置ssh开机启动与环境变量文件生效
   编辑 .bashrc文件 容器启动会执行此文件 注意如此文件在/root目录不执行可将此文件cp到根目录
   ssh免密登录自行百度
   在文件中追加

service ssh start >>/root/start_ssh.log
export JAVA_HOME=/root/jdk-11.0.15
export HADOOP_HOME=/root/hadoop-3.2.3
export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

2.4 配置hadoop个配置文件

vim /root/hadoop-3.2.3/etc/hadoop/hadoop-env.sh

JAVA_HOME=jdk路径
 #hdfs
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
 # yarn
export YARN_RESOURCEMANAGER_USER=root
export HADOOP_SECURE_DN_USER=yarn
export YARN_NODEMANAGER_USER=root

============================================
vim /root/hadoop-3.2.3/etc/hadoop/core-site.xml

<configuration>
    <property>
      <name>fs.defaultFSname>
      <value>hdfs://namenode:9000value>
    property>
    <property>
        <name>hadoop.tmp.dirname>
        <value>/opt/hadoop/datavalue>
    property>
    <property>
	    <name>hadoop.http.staticuser.username>
	    <value>rootvalue>
    property>
configuration>

============================================
vim /root/hadoop-3.2.3/etc/hadoop/yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-servicesname>
        <value>mapreduce_shufflevalue>
    property>
    <property>
        <name>yarn.resourcemanager.hostnamename>
        <value>resourcemanagervalue>
    property>
configuration>

============================================
vim /root/hadoop-3.2.3/etc/hadoop/mapred-site.xml

<configuration>
    <property>
      <name>mapreduce.framework.namename>
      <value>yarnvalue>
    property>
    <property>
        <name>mapreduce.jobhistory.addressname>
        <value>jobhistory:10020value>
    property>
    <property>
        <name>mapreduce.jobhistory.webapp.addressname>
        <value>jobhistory:19888value>
    property>
configuration>

============================================
vim /root/hadoop-3.2.3/etc/hadoop/hdfs-site.xml

<configuration>
    <property>
      <name>dfs.replicationname>
      <value>2value>
    property>
    <property>
       <name>dfs.datanode.data.dirname>
       <value>file:///hadoop/hdfs/datavalue>
    property>
    <property>
        <name>dfs.namenode.secondary.http-addressname>
        <value>secondarynamenode:9868value>
    property>
    <property>
        <name>dfs.namenode.http-addressname>
        <value>namenode:9870value>
    property>
configuration>

============================================
vim /root/hadoop-3.2.3/etc/hadoop/workers

slave2
slave1
namenode
secondarynamenode

2.5 打包镜像
docker commit 容器id 镜像名称:TAG
3. 创建网络
docker network create hadoop_nw
4. 编写docker-compose.yml文件

version: "3.9"
services:
  namenode:
    ports:
      - "9870:9870"
      - "9000:9000"
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "namenode"
    tty: true
  secondarynamenode:
    ports:
      - "9868:9868"
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "secondarynamenode"
    tty: true
  resourcemanager:
    ports:
      - "8032:8032"
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "resourcemanager"
    tty: true
  jobhistory:
    ports:
      - "19888:19888"
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "jobhistory"
    tty: true
  slave1:
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "slave1"
    tty: true
  slave2:
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "slave2"
    tty: true
  client:
    image: "myhadoop:v2"
    networks:
      - hadoop_nw
    container_name: "client"
    tty: true
networks:
  hadoop_nw:
    external: true

5. 启动 docker compose up -d
6. 进入任意容器启动集群

hdfs namenode -format
start-dfs.sh
start-yarn.sh
mr-jobhistory-daemon.sh --config $HADOOP_CONF_DIR start historyserver

7. 验证 按照对外暴露的接口可访问验证如：http://localhost:9870/