docker-compose部署zookeeper&kafka集群
环境说明
Zookeeper集群
| IP | 节点目录名 | 容器名 | client port | follower port | election port |
|---|---|---|---|---|---|
| 192.168.78.155 | zookeeper1 | zookeeper1 | 2181 | 2888 | 3888 |
| 192.168.78.156 | zookeeper2 | zookeeper2 | 2181 | 2888 | 3888 |
| 192.168.78.157 | zookeeper3 | zookeeper3 | 2181 | 2888 | 3888 |
Kafka集群
| IP | 节点目录名 | 容器名 | 占用端口 |
|---|---|---|---|
| 192.168.78.155 | kafka1 | kafka1 | 9092 |
| 192.168.78.156 | kafka2 | kafka2 | 9092 |
| 192.168.78.157 | kafka3 | kafka3 | 9092 |
| 192.168.78.155 | kafka-manager | kafka-manager | 9000 |
每台服务器上安装docker-compose
[root@kafka1 ~]# sudo curl -L "https://github.com/docker/compose/releases/download/1.27.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
[root@kafka1 ~]# sudo chmod +x /usr/local/bin/docker-compose
[root@kafka1 ~]# docker-compose version
docker-compose version 1.27.1, build 509cfb99
docker-py version: 4.3.1
CPython version: 3.7.7
OpenSSL version: OpenSSL 1.1.0l 10 Sep 2019
根据环境部署zookeeper，kafka，kafka-manager
Zookeeper集群使用
- 请确保所布署的 1~3 台服务器网络可以ping通
- 确保每台主机的2181\2888\3888端口未占用
zookeeper1
准备挂载目录
[root@kafka1 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper1
[root@kafka1 ~]# cd /root/zk_kafka_cluster/zookeeper1
[root@kafka1 zookeeper1]# mkdir {data,datalog}
[root@kafka1 zookeeper1]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
zookeeper1:
image: zookeeper #镜像
restart: always # 重启
container_name: zookeeper1 #容器名
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- /root/zk_kafka_cluster/zookeeper1/data:/data
- /root/zk_kafka_cluster/zookeeper1/datalog:/datalog
environment:
ZOO_MY_ID: 1 # id
ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
network_mode: "host"
启动zookeepr1
[root@kafka1 zookeeper1]# docker-compose up -d
查看状态
[root@kafka1 zookeeper1]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
61de533366f5 zookeeper "/docker-entrypoint.…" 11 minutes ago Up 11 minutes zookeeper1
[root@kafka1 zookeeper1]# docker exec -it zookeeper1 /bin/bash
root@kafka1:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
zookeeper2
准备挂载目录
[root@kafka2 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper2
[root@kafka2 ~]# cd /root/zk_kafka_cluster/zookeeper2
[root@kafka2 zookeeper2]# mkdir {data,datalog}
[root@kafka2 zookeeper2]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
zookeeper2:
image: zookeeper #镜像
restart: always # 重启
container_name: zookeeper2 #容器名
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- /root/zk_kafka_cluster/zookeeper2/data:/data
- /root/zk_kafka_cluster/zookeeper2/datalog:/datalog
environment:
ZOO_MY_ID: 2 # id
ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
network_mode: "host"
启动zookeeper2
[root@kafka2 zookeeper2]# docker-compose up -d
查看状态
[root@kafka2 zookeeper2]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
316ba6fec210 zookeeper "/docker-entrypoint.…" 12 minutes ago Up 12 minutes zookeeper2
[root@kafka2 zookeeper2]# docker exec -it zookeeper2 /bin/bash
root@kafka2:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader
zookeeper3
准备挂载目录
[root@kafka3 ~]# mkdir -p /root/zk_kafka_cluster/zookeeper3
[root@kafka3 ~]# cd /root/zk_kafka_cluster/zookeeper3
[root@kafka3 zookeeper3]# mkdir {data,datalog}
[root@kafka3 zookeeper3]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
zookeeper3:
image: zookeeper #镜像
restart: always # 重启
container_name: zookeeper3 #容器名
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- /root/zk_kafka_cluster/zookeeper3/data:/data
- /root/zk_kafka_cluster/zookeeper3/datalog:/datalog
environment:
ZOO_MY_ID: 3 # id
ZOO_SERVERS: server.1=192.168.78.155:2888:3888;2181 server.2=192.168.78.156:2888:3888;2181 server.3=192.168.78.157:2888:3888;2181
network_mode: "host"
启动zookeeper3
[root@kafka3 zookeeper3]# docker-compose up -d
查看状态
[root@kafka3 zookeeper3]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
eebed4fe0ccd zookeeper "/docker-entrypoint.…" 12 minutes ago Up 12 minutes zookeeper3
[root@kafka3 zookeeper3]# docker exec -it zookeeper3 /bin/bash
root@kafka3:/apache-zookeeper-3.6.1-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
Kafka集群使用
使用前确保各主机可以互相ping通
确保zookeeper的服务列表与各对应的zookeeper的ip与客户端口相同
确保zookeeper集群启动
确保每台主机对应的占用端口号不被占用
kafka1对应9092、kafka2对应9092、kafka3对应9092、kafka-manager对应9000-
分别对每一台kafkax所在的主机修改
/etc/hosts，例192.168.78.155 kafka1 192.168.78.156 kafka2 192.168.78.157 kafka3 启动脚本中没有启动
kafka-manager，有需要请自行启动。为了匹配kafka的版本，使用时设置2.1.1即可。
kafka1
准备挂载目录
[root@kafka1 ~]# cd zk_kafka_cluster/
[root@kafka1 zk_kafka_cluster]# mkdir -p kafka1/logs
[root@kafka1 zk_kafka_cluster]# cd kafka1
[root@kafka1 kafka1]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
kafka1:
image: wurstmeister/kafka # 镜像
restart: always #重启
container_name: kafka1 #容器名
ports:
- 9092:9092
- 9999:9999
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.155:9092 #绑定发布订阅的端口
KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
KAFKA_BROKER_ID: 0 #kafka的broker.id，区分不同broker
KAFKA_LISTENERS: PLAINTEXT://192.168.78.155:9092 #绑定监听9092端口
JMX_PORT: 9999 # jmx
KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
volumes:
- /etc/localtime:/etc/localtime
- /root/zk_kafka_cluster/kafka1/logs:/kafka
network_mode: "host"
启动kafka1
[root@kafka1 kafka1]# docker-compose up -d
kafka2
准备挂载目录
[root@kafka2 ~]# cd zk_kafka_cluster/
[root@kafka2 zk_kafka_cluster]# mkdir -p kafka2/logs
[root@kafka2 zk_kafka_cluster]# cd kafka2
[root@kafka2 kafka2]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
kafka2:
image: wurstmeister/kafka # 镜像
restart: always #重启
container_name: kafka2 #容器名
ports:
- 9092:9092
- 9999:9999
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.156:9092 #绑定发布订阅的端口
KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
KAFKA_BROKER_ID: 1 #kafka的broker.id，区分不同broker
KAFKA_LISTENERS: PLAINTEXT://192.168.78.156:9092 #绑定监听9092端口
JMX_PORT: 9999 # jmx
KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
volumes:
- /etc/localtime:/etc/localtime
- /root/zk_kafka_cluster/kafka2/logs:/kafka
network_mode: "host"
启动kafka2
[root@kafka2 kafka2]# docker-compose up -d
kafka3
准备挂载目录
[root@kafka3 ~]# cd zk_kafka_cluster/
[root@kafka3 zk_kafka_cluster]# mkdir -p kafka3/logs
[root@kafka3 zk_kafka_cluster]# cd kafka3
[root@kafka3 kafka3]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
kafka3:
image: wurstmeister/kafka # 镜像
restart: always #重启
container_name: kafka3 #容器名
ports:
- 9092:9092
- 9999:9999
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.78.157:9092 #绑定发布订阅的端口
KAFKA_ZOOKEEPER_CONNECT: 192.168.78.155:2181,192.168.78.156:2181,192.168.78.157:2181 #连接zookeeper的服务地址
KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
KAFKA_BROKER_ID: 2 #kafka的broker.id，区分不同broker
KAFKA_LISTENERS: PLAINTEXT://192.168.78.157:9092 #绑定监听9092端口
JMX_PORT: 9999 # jmx
KAFKA_MESSAGE_MAX_BYTES: 2000000 #单条消息最大字节数
volumes:
- /etc/localtime:/etc/localtime
- /root/zk_kafka_cluster/kafka3/logs:/kafka
network_mode: "host"
启动kafka3
[root@kafka3 kafka3]# docker-compose up -d
kafka-manager
[root@kafka1 ~]# cd zk_kafka_cluster/
[root@kafka1 zk_kafka_cluster]# mkdir kafka-manager
[root@kafka1 zk_kafka_cluster]# cd kafka-manager/
[root@kafka1 kafka-manager]# vim docker-compose.yml
编写docker-compose.yml文件
version: '3.8'
services:
kafka-manager:
image: hlebalbau/kafka-manager:stable
restart: always
container_name: kafka-manager
ports:
- 9000:9000
environment:
ZK_HOSTS: 192.168.68.155:2181,192.168.78.156:2181,192.168.78.157:2181
KAFKA_BROKERS: 192.168.68.155:9092,192.168.68.156:9092,192.168.68.157:9092
APPLICATION_SECRET: "random-secret"
KAFKA_MANAGER_AUTH_ENABLED: "true" # 开启验证
KAFKA_MANAGER_USERNAME: "admin" # 用户名
KAFKA_MANAGER_PASSWORD: "admin" # 密码
KM_ARGS: -Djava.net.preferIPv4Stack=true
network_mode: "host"
启动kafka-manager
[root@kafka1 kafka-manager]# docker-compose up -d
访问192.168.78.155:9000
账号：admin
密码：admin
手动添加集群即可