Docker容器部署kafka&&zookeeper

一、拉取镜像

docker pull wurstmeister/zookeeper
docker pull wurstmeister/kafka

二、 启动容器

# 启动zookeeper
docker run -d --name zookeeper -p 2181:2181 -t wurstmeister/zookeeper
# 启动kafka  192.168.0.100改为自己机器实际IP
docker run -d --name kafka --publish 9092:9092  --link zookeeper --env KAFKA_BROKER_ID=0 --env KAFKA_ZOOKEEPER_CONNECT=192.168.0.100:2181  --env KAFKA_ADVERTISED_HOST_NAME=192.168.0.100  --env KAFKA_ADVERTISED_PORT=9092   wurstmeister/kafka

三、基础命令

进入容器

docker exec -it kafka /bin/bash

Topic创建

#创建一个名为 test 的 Topic，该 Topic 包含一个分区和一个 Replica。
# 进入kafka容器中执行
kafka-topics.sh --create --zookeeper 192.168.0.100:2181 --replication-factor 1 --partitions 1 --topic test
# 不进入直接执行
docker exec kafka kafka-topics.sh --create --zookeeper 192.168.0.100:2181 --replication-factor 1 --partitions 1 --topic test

Topic查询

# 进入kafka容器中执行
kafka-topics.sh --list --zookeeper 192.168.0.100:2181
# 不进入直接执行
docker exec kafka kafka-topics.sh --list --zookeeper 192.168.0.100:2181

命令行消息生产者创建

# 进入kafka容器中执行
kafka-console-producer.sh --broker-list 192.168.0.100:9092 --topic test
# 不进入直接执行
docker exec -it kafka kafka-console-producer.sh --broker-list 192.168.0.100:9092 --topic test

命令行消息消费者创建

# 进入kafka容器中执行
kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9092 --topic test --from-beginning
# 不进入直接执行
docker exec -it kafka kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9092 --topic test --from-beginning

Topic删除

# 进入kafka容器中执行
kafka-topics.sh --zookeeper 192.168.0.100:2181 --delete --topic test
# 不进入直接执行
docker exec kafka kafka-topics.sh --zookeeper 192.168.0.100:2181 --delete --topic test

四、Broker集群搭建&&容错测试

创建zookeeper和三个kafka实例

docker run -d --name zookeeper -p 2181:2181 -t wurstmeister/zookeeper

docker run -d --name kafka0 -p 9092:9092 -e KAFKA_BROKER_ID=0 -e KAFKA_ZOOKEEPER_CONNECT=192.168.0.100:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.0.100:9092 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 -t wurstmeister/kafka

docker run -d --name kafka1 -p 9093:9093 -e KAFKA_BROKER_ID=1 -e KAFKA_ZOOKEEPER_CONNECT=192.168.0.100:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.0.100:9093 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9093 -t wurstmeister/kafka

docker run -d --name kafka2 -p 9094:9094 -e KAFKA_BROKER_ID=2 -e KAFKA_ZOOKEEPER_CONNECT=192.168.0.100:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.0.100:9094 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9094 -t wurstmeister/kafka

创建一个拥有3副本的topic

#创建topic
docker exec kafka0 kafka-topics.sh --create --zookeeper 192.168.0.100:2181 --replication-factor 3 --partitions 1 --topic TestTopic
#查看topic
docker exec kafka0 kafka-topics.sh --topic TestTopic --describe --zookeeper 192.168.0.100:2181

#可以看到，当前Leader是Broker0
Topic: TestTopic        PartitionCount: 1       ReplicationFactor: 3    Configs:
        Topic: TestTopic        Partition: 0    Leader: 0       Replicas: 0,2,1 Isr: 0,2,1

集群测试

#在Broker0上运行一个生产者，Broker1、2上分别运行一个消费者
kafka-console-producer.sh --broker-list 192.168.0.100:9092 --topic TestTopic

kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9093 --topic TestTopic --from-beginning

kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9094 --topic TestTopic --from-beginning

#Broker0命令行输入消息，观察Broker1、2是否收到

容错测试

#在Broker0上运行一个生产者，Broker1上运行一个消费者
kafka-console-producer.sh --broker-list 192.168.0.100:9092 --topic TestTopic

kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9093 --topic TestTopic --from-beginning

#停掉Broker0，Broker2上运行一个消费者，观察Broker2能否收到消息
docker stop kafka0

kafka-console-consumer.sh --bootstrap-server 192.168.0.100:9094 --topic TestTopic --from-beginning

五、docker-compose创建kafka集群

docker-compose-kafka-cluster.yml 文件

version: '3'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"

  kafka1:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.0.100
      KAFKA_CREATE_TOPICS: TestComposeTopic:4:3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.0.100:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
    container_name: kafka01
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

  kafka2:
    image: wurstmeister/kafka
    ports:
      - "9093:9093"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.0.100
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 2
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.0.100:9093
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093
    container_name: kafka02
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

  kafka3:
    image: wurstmeister/kafka
    ports:
      - "9094:9094"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.0.100
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 3
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.0.100:9094
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094
    container_name: kafka03
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

启动命令

docker-compose -f docker-compose-kafka-cluster.yml up