使用docker搭建zookeeper和kafka集群

一、镜像选择

Zookeeper和Kafka集群分别运行在不同的容器中 zookeeper官方镜像，版本latest
kafka采用fogsyio/kafka镜像,版本latest kafuka
manager采用scjtqs/kafka-manager镜像,版本latest

二、集群规划

hostname	Ip addr	port	listener
zook1	172.20.10.11	2184:2181
zook2	172.20.10.12	2185:2181
zook3	172.20.10.13	2186:2181
kafka1	172.20.10.14	内部9092:9092，外部9192:9192	kafka1
kafka2	172.20.10.15	内部9093:9093，外部9193:9193	kafka2
Kafka3	172.20.10.16	内部9094:9094，外部9194:9194	Kafka3
宿主机root OSX	172.20.10.2
kafuka manager	172.20.10.10	9000:9000

三、实现目标

kafka集群在docker网络中可用，和zookeeper处于同一网络
宿主机可以访问zookeeper集群和kafka的broker list
docker重启时集群自动重启
集群的数据文件映射到宿主机器目录中
使用yml文件和$ docker-compose up -d命令创建或重建集群

四、搭建集群

1.搭建zookeeper集群

命令对照

命令	解释
docker-compose up	启动所有容器
docker-compose up -d	后台启动并运行所有容器
docker-compose up --no-recreate -d	不重新创建已经停止的容器
docker-compose up -d test2	只启动test2这个容器
docker-compose stop	停止容器
docker-compose start	启动容器
docker-compose down	停止并销毁容器

1.1新建docker网络

$ docker network create zkkf-net --subnet 172.20.10.0/16
$ docker network ``ls

1.2编写 docker-compose-zookeeper.yml 脚本

version: '3.4'

services:
  zook1:
    image: zookeeper:latest
    restart: always
    hostname: zook1
    container_name: zook1 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2184:2181 #将本容器的zookeeper默认端口号映射出去
    volumes: # 挂载数据卷
    - "/Users/tangyunhang/docker/zookeeper/zook1/data:/data"
    - "/Users/tangyunhang/docker/zookeeper/zook1/datalog:/datalog"
    environment:
        ZOO_MY_ID: 1  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        zkkf-net:
            ipv4_address: 172.20.10.11
            
  zook2:   
    image: zookeeper:latest
    restart: always
    hostname: zook2
    container_name: zook2 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2185:2181 #将本容器的zookeeper默认端口号映射出去
    volumes:
    - "/Users/tangyunhang/docker/zookeeper/zook2/data:/data"
    - "/Users/tangyunhang/docker/zookeeper/zook2/datalog:/datalog"
    environment:
        ZOO_MY_ID: 2  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        zkkf-net:
            ipv4_address: 172.20.10.12
            
  zook3:   
    image: zookeeper:latest
    restart: always
    hostname: zook3
    container_name: zook3 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2185:2181 #将本容器的zookeeper默认端口号映射出去
    volumes:
    - "/Users/tangyunhang/docker/zookeeper/zook3/data:/data"
    - "/Users/tangyunhang/docker/zookeeper/zook3/datalog:/datalog"
    environment:
        ZOO_MY_ID: 3  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        zkkf-net:
            ipv4_address: 172.20.10.13

networks:
  zkkf-net:
    external:
      name: zkkf-net

1.3执行脚本

docker-compose -f docker-compose-zookeeper.yml up -d

1.4验证zookeeper集群

tangyunhang@tangyunhangdeMacBook-Pro zookeeper % docker exec zook1 zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
#成功 zook1 ：follower， zook2 ：follower， zook3 ：leader

2.搭建Kafka集群

2.1编写 docker-compose-kafka.yml 脚本

kafka 内外端口设置参考：https://www.jianshu.com/p/26495e334613

mkdir -p kafka
vi docker-compose-kafka.yml

version: '2'

services:
  kafka1:
    image: fogsyio/kafka:arm64v8-2.2.0
    restart: always
    hostname: kafka1
    container_name: kafka1
    ports:
      - 9092:9092
      - 9192:9192
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9192
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://:9092,OUTSIDE://:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:9092,OUTSIDE://localhost:9192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
    volumes:
      - /Users/tangyunhang/docker/kafka/kafka1/logs:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      zkkf-net:
        ipv4_address: 172.20.10.14

  kafka2:
    image: fogsyio/kafka:arm64v8-2.2.0
    restart: always
    hostname: kafka2
    container_name: kafka2
    ports:
      - 9093:9093
      - 9193:9193
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9193
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://:9092,OUTSIDE://:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka2:9093,OUTSIDE://localhost:9193
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
    volumes:
      - /Users/tangyunhang/docker/kafka/kafka2/logs:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      zkkf-net:
        ipv4_address: 172.20.10.15

  kafka3:
    image: fogsyio/kafka:arm64v8-2.2.0
    restart: always
    hostname: kafka3
    container_name: kafka3
    ports:
      - 9094:9094
      - 9194:9194
    environment:
      KAFKA_LISTENERS: INSIDE://:9094,OUTSIDE://:9194
      #KAFKA_ADVERTISED_LISTENERS=INSIDE://:9092,OUTSIDE://:9094
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka3:9094,OUTSIDE://localhost:9194
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
    volumes:
      - /Users/tangyunhang/docker/kafka/kafka3/logs:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      zkkf-net:
        ipv4_address: 172.20.10.16

  kafka-manager:
    image: scjtqs/kafka-manager:latest
    restart: always
    hostname: kafka-manager
    container_name: kafka-manager
    ports:
      - 9000:9000
    links:  # 连接本compose文件创建的container
      - kafka1
      - kafka2
      - kafka3
    external_links:  # 连接本compose文件以外的container
      - zook1
      - zook2
      - zook3
    environment:
      ZK_HOSTS: zook1:2181,zook2:2181,zook3:2181
      KAFKA_BROKERS: kafka1:9093,kafka2:9094,kafka3:9095
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true
    networks:
      zkkf-net:
        ipv4_address: 172.20.10.10

networks:
  zkkf-net:
    external:
      name: zkkf-net

2.2执行脚本

docker-compose -f docker-compose-kafka.yml up -d

3.验证kafka安装情况

kafka-manager的管理页面，访问路径是，宿主机ip:9000；

点击刚刚添加的集群，可以看到，集群中有三个节点