Docker搭建Zookeeper和Kafka集群

安装Docker-compose

创建一个带子网的专有网络

创建一个叫zoo_kafka的网络,子网网段172.23.0.10/20

root@localhost ]# docker network create --subnet=172.23.0.10/20 zoo_kafka
root@localhost ]# docker network ls
NETWORK ID          NAME                DRIVER              SCOPE
4fad358ef523        bridge              bridge              local
2ea1ad136fa2        host                host                local
a66aec056df4        kafka_default       bridge              local
9a4fbf8d993d        none                null                local
d584dd2084c7        zoo_kafka           bridge              local

创建Zookeeper集群

创建文件夹

root@localhost ]# mkdir zookeeper
root@localhost ]# cd zookeeper

编写docker-compose.yml文件

root@localhost ]# touch docker-compose.yml
root@localhost ]# vim docker-compose.yml

内容如下:

version: '2'
services:
  zoo1:
    image: zookeeper:3.4 # 镜像名称
    restart: always # 当发生错误时自动重启
    hostname: zoo1
    container_name: zoo1
    privileged: true
    ports: # 端口
      - 2181:2181
    volumes: # 挂载数据卷
      - ./zoo1/data:/data
      - ./zoo1/datalog:/datalog 
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 1 # 节点ID
      ZOO_PORT: 2181 # zookeeper端口号
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 # zookeeper节点列表
    networks:
      default:
        ipv4_address: 172.23.0.11

  zoo2:
    image: zookeeper:3.4
    restart: always
    hostname: zoo2
    container_name: zoo2
    privileged: true
    ports:
      - 2182:2181
    volumes:
      - ./zoo2/data:/data
      - ./zoo2/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 2
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.12

  zoo3:
    image: zookeeper:3.4
    restart: always
    hostname: zoo3
    container_name: zoo3
    privileged: true
    ports:
      - 2183:2181
    volumes:
      - ./zoo3/data:/data
      - ./zoo3/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 3
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.13

networks:
  default:
    external:
      name: zoo_kafka

启动并验证

• 后台启动,并查看容器

root@localhost zookeeper]# docker-compose up -d
Starting zoo3 ... done
Starting zoo2 ... done
Starting zoo1 ... done
[root@localhost zookeeper]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                        NAMES
125a78286d6b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
4998e54b7140        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
5eec0034d73b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3

• 验证zookeeper集群
  这里看到zoo3为leader,zoo1和zoo2为follower

[root@localhost zookeeper]# docker exec zoo1 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: follower
[root@localhost zookeeper]# docker exec zoo2 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: follower
[root@localhost zookeeper]# docker exec zoo3 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: leader

创建Kafka集群

创建文件夹

创建文件夹

root@localhost ]# mkdir kafka
root@localhost ]# cd kafka

编写docker-compose.yml文件

root@localhost ]# touch docker-compose.yml
root@localhost ]# vim docker-compose.yml

内容如下:

version: '2'

services:
  broker1:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker1
    container_name: broker1
    privileged: true
    ports:
      - "9091:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: PLAINTEXT://broker1:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:9092
      KAFKA_ADVERTISED_HOST_NAME: broker1
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker1:/kafka/kafka\-logs\-broker1
    external_links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.14

  broker2:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker2
    container_name: broker2
    privileged: true
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: PLAINTEXT://broker2:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:9092
      KAFKA_ADVERTISED_HOST_NAME: broker2
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker2:/kafka/kafka\-logs\-broker2
    external_links:  # 连接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.15

  broker3:
    image: wurstmeister/kafka:2.11-0.11.0.3
    restart: always
    hostname: broker3
    container_name: broker3
    privileged: true
    ports:
      - "9093:9092"
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: PLAINTEXT://broker3:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:9092
      KAFKA_ADVERTISED_HOST_NAME: broker3
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      JMX_PORT: 9988
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./broker3:/kafka/kafka\-logs\-broker3
    external_links:  # 连接本compose文件以外的container
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.16

  kafka-manager:
    image: sheepkiller/kafka-manager:latest
    restart: always
    container_name: kafka-manager
    hostname: kafka-manager
    ports:
      - "9000:9000"
    links:            # 连接本compose文件创建的container
      - broker1
      - broker2
      - broker3
    external_links:   # 连接本compose文件以外的container
      - zoo1
      - zoo2
      - zoo3
    environment:
      ZK_HOSTS: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
      KAFKA_BROKERS: broker1:9092,broker2:9092,broker3:9092
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true
    networks:
      default:
        ipv4_address: 172.23.0.10

networks:
  default:
    external:   # 使用已创建的网络
      name: zoo_kafka

启动并验证

[root@localhost kafka]# docker-compose up -d
Creating broker1 ... done
Creating broker3 ... done
Creating broker2 ... done
Creating kafka-manager ... done
[root@localhost kafka]# docker ps
CONTAINER ID        IMAGE                              COMMAND                  CREATED             STATUS              PORTS                                        NAMES
8140b9c9eb07        sheepkiller/kafka-manager:latest   "./start-kafka-manag…"   4 minutes ago       Up 4 minutes        0.0.0.0:9000->9000/tcp                       kafka-manager
5df60f08f5aa        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9093->9092/tcp                       broker3
4d17d490faa7        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9092->9092/tcp                       broker2
07206f603d20        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9091->9092/tcp                       broker1
125a78286d6b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
4998e54b7140        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
5eec0034d73b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3