Docker容器部署kafka&&zookeeper
一、拉取镜像
docker pull wurstmeister/zookeeper
docker pull wurstmeister/kafka
二、 启动容器
启动zookeeper
docker run -d --name zookeeper -p 2181:2181 -t wurstmeister/zookeeper
启动kafka 10.234.63.73改为自己机器实际IP
docker run -d --name kafka --publish 9092:9092 --link zookeeper --env KAFKA_BROKER_ID=0 --env KAFKA_ZOOKEEPER_CONNECT=10.234.63.73:2181 --env KAFKA_ADVERTISED_HOST_NAME=10.234.63.73 --env KAFKA_ADVERTISED_PORT=9092 wurstmeister/kafka
三、基础命令
进入容器
docker exec -it kafka /bin/bash
Topic创建
#创建一个名为 test 的 Topic，该 Topic 包含一个分区和一个 Replica。
进入kafka容器中执行
kafka-topics.sh --create --zookeeper 10.234.63.73:2181 --replication-factor 1 --partitions 1 --topic test
不进入直接执行
docker exec kafka kafka-topics.sh --create --zookeeper 10.234.63.73:2181 --replication-factor 1 --partitions 1 --topic test
Topic查询
进入kafka容器中执行
kafka-topics.sh --list --zookeeper 10.234.63.73:2181
不进入直接执行
docker exec kafka kafka-topics.sh --list --zookeeper 10.234.63.73:2181
命令行消息生产者创建
进入kafka容器中执行
kafka-console-producer.sh --broker-list 10.234.63.73:9092 --topic test
不进入直接执行
docker exec -it kafka kafka-console-producer.sh --broker-list 10.234.63.73:9092 --topic test
命令行消息消费者创建
进入kafka容器中执行
kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9092 --topic test --from-beginning
不进入直接执行
docker exec -it kafka kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9092 --topic test --from-beginning
Topic删除
进入kafka容器中执行
kafka-topics.sh --zookeeper 10.234.63.73:2181 --delete --topic test
不进入直接执行
docker exec kafka kafka-topics.sh --zookeeper 10.234.63.73:2181 --delete --topic test
四、Broker集群搭建&&容错测试
创建zookeeper和三个kafka实例
docker run -d --name zookeeper -p 2181:2181 -t wurstmeister/zookeeper
docker run -d --name kafka0 -p 9092:9092 -e KAFKA_BROKER_ID=0 -e KAFKA_ZOOKEEPER_CONNECT=10.234.63.73:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.234.63.73:9092 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 -t wurstmeister/kafka
docker run -d --name kafka1 -p 9093:9093 -e KAFKA_BROKER_ID=1 -e KAFKA_ZOOKEEPER_CONNECT=10.234.63.73:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.234.63.73:9093 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9093 -t wurstmeister/kafka
docker run -d --name kafka2 -p 9094:9094 -e KAFKA_BROKER_ID=2 -e KAFKA_ZOOKEEPER_CONNECT=10.234.63.73:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.234.63.73:9094 -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9094 -t wurstmeister/kafka
创建一个拥有3副本的topic
#创建topic
docker exec kafka0 kafka-topics.sh --create --zookeeper 10.234.63.73:2181 --replication-factor 3 --partitions 1 --topic TestTopic
#查看topic
docker exec kafka0 kafka-topics.sh --topic TestTopic --describe --zookeeper 10.234.63.73:2181
#可以看到，当前Leader是Broker0
Topic: TestTopic PartitionCount: 1 ReplicationFactor: 3 Configs:
Topic: TestTopic Partition: 0 Leader: 0 Replicas: 0,2,1 Isr: 0,2,1
集群测试
#在Broker0上运行一个生产者，Broker1、2上分别运行一个消费者
kafka-console-producer.sh --broker-list 10.234.63.73:9092 --topic TestTopic
kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9093 --topic TestTopic --from-beginning
kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9094 --topic TestTopic --from-beginning
#Broker0命令行输入消息，观察Broker1、2是否收到
容错测试
#在Broker0上运行一个生产者，Broker1上运行一个消费者
kafka-console-producer.sh --broker-list 10.234.63.73:9092 --topic TestTopic
kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9093 --topic TestTopic --from-beginning
#停掉Broker0，Broker2上运行一个消费者，观察Broker2能否收到消息
docker stop kafka0
kafka-console-consumer.sh --bootstrap-server 10.234.63.73:9094 --topic TestTopic --from-beginning
五、docker-compose创建kafka集群
docker-compose-kafka-cluster.yml 文件
version: ‘3’
services:
zookeeper:
image: wurstmeister/zookeeper
container_name: zookeeper
ports:
- “2181:2181”
kafka1:
image: wurstmeister/kafka
ports:
- “9092:9092”
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.234.63.73
KAFKA_CREATE_TOPICS: TestComposeTopic:4:3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_BROKER_ID: 1
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.234.63.73:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
container_name: kafka01
volumes:
- /var/run/docker.sock:/var/run/docker.sock
kafka2:
image: wurstmeister/kafka
ports:
- “9093:9093”
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.234.63.73
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_BROKER_ID: 2
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.234.63.73:9093
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093
container_name: kafka02
volumes:
- /var/run/docker.sock:/var/run/docker.sock
kafka3:
image: wurstmeister/kafka
ports:
- “9094:9094”
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.234.63.73
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_BROKER_ID: 3
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.234.63.73:9094
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094
container_name: kafka03
volumes:
- /var/run/docker.sock:/var/run/docker.sock
启动命令
docker-compose -f docker-compose-kafka-cluster.yml up