docker部暑zookeeper_3.6\kafka_2.4集群（支持amd64和arm64）
1、安装zookeeper
| IP | 角色 | 备注 |
|---|---|---|
| 192.168.11.192 | server | 节点1 |
| 192.168.11.193 | server | 节点2 |
| 192.168.11.194 | server,client,ui | 节点3 |
2、配置文件说明
zoo.conf
#4lw.commands.whitelist=* #访问白名单（3.6以上）
4lw.commands.whitelist=mntr,ruok
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
quorum.auth.enableSasl=true # 打开sasl开关, 默认是关的
quorum.auth.learnerRequireSasl=true # ZK做为leaner的时候, 会发送认证信息
quorum.auth.serverRequireSasl=true # 设置为true的时候,learner连接的时候需要发送认证信息,否则拒绝
quorum.auth.learner.loginContext=QuorumLearner # JAAS 配置里面的 Context 名字
quorum.auth.server.loginContext=QuorumServer # JAAS 配置里面的 Context 名字
quorum.cnxn.threads.size=20 # 建议设置成ZK节点的数量乘2
------
zk_server_jaas.conf
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin" # zookeeper之间的认证用户名
password="admin" # zookeeper之间的认证密码
user_kafka="admin" # 为kafka服务创建账号密码：用户名kafka，密码admin
user_producer="admin"; # 根据实际情况增加用户，这里增加一个用户名为producer,密码为admin的用户
};
QuorumServer {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_zookeeper="zookeeper@password"; # 用户名为zookeeper，密码为zookeeper@password
};
QuorumLearner {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="zookeeper"
password="zookeeper@password";
};
# 节点192
mkdir -p /data/zookeeper/{data,conf,log}
echo '192' > /data/zookeeper/data/myid
#zookeeper配置文件
cat > /data/zookeeper/conf/zoo.cfg << 'EOF'
#4lw.commands.whitelist=*
4lw.commands.whitelist=mntr,ruok
clientPort=2181
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
server.192=192.168.11.192:2888:3888
server.193=192.168.11.193:2888:3888
server.194=192.168.11.194:2888:3888
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
quorum.auth.enableSasl=true
quorum.auth.learnerRequireSasl=true
quorum.auth.serverRequireSasl=true
quorum.auth.learner.loginContext=QuorumLearner
quorum.auth.server.loginContext=QuorumServer
quorum.cnxn.threads.size=6
EOF
#zookeeper的sasl配置
cat > /data/zookeeper/conf/zk_server_jaas.conf << 'EOF'
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin"
password="admin"
user_kafka="admin"
user_producer="admin";
};
QuorumServer {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_zookeeper="zookeeper@password";
};
QuorumLearner {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="zookeeper"
password="zookeeper@password";
};
EOF
cat > /data/zookeeper/startup.sh << 'EOF'
docker run -d \
--network host \
--restart=always \
-e "SERVER_JVMFLAGS=-Djava.security.auth.login.config=/conf/zk_server_jaas.conf" \
-v /data/zookeeper/data:/data/zookeeper/data \
-v /data/zookeeper/conf:/conf \
-v /etc/localtime:/etc/localtime \
--name zookeeper \
zookeeper:3.6.3
EOF
bash /data/zookeeper/startup.sh
# 节点193
mkdir -p /data/zookeeper/{data,conf,log}
echo '193' > /data/zookeeper/data/myid
cat > /data/zookeeper/conf/zoo.cfg << 'EOF'
#4lw.commands.whitelist=*
4lw.commands.whitelist=mntr,ruok
clientPort=2181
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
server.192=192.168.11.192:2888:3888
server.193=192.168.11.193:2888:3888
server.194=192.168.11.194:2888:3888
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
quorum.auth.enableSasl=true
quorum.auth.learnerRequireSasl=true
quorum.auth.serverRequireSasl=true
quorum.auth.learner.loginContext=QuorumLearner
quorum.auth.server.loginContext=QuorumServer
quorum.cnxn.threads.size=6
EOF
#zookeeper的sasl配置
cat > /data/zookeeper/conf/zk_server_jaas.conf << 'EOF'
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin"
password="admin"
user_kafka="admin"
user_producer="admin";
};
QuorumServer {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_zookeeper="zookeeper@password";
};
QuorumLearner {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="zookeeper"
password="zookeeper@password";
};
EOF
cat > /data/zookeeper/startup.sh << 'EOF'
docker run -d \
--network host \
--restart=always \
-e "SERVER_JVMFLAGS=-Djava.security.auth.login.config=/conf/zk_server_jaas.conf" \
-v /data/zookeeper/data:/data/zookeeper/data \
-v /data/zookeeper/conf:/conf \
-v /etc/localtime:/etc/localtime \
--name zookeeper \
zookeeper:3.6.3
EOF
bash /data/zookeeper/startup.sh
# 节点194
mkdir -p /data/zookeeper/{data,conf,log}
echo '194' > /data/zookeeper/data/myid
cat > /data/zookeeper/conf/zoo.cfg << 'EOF'
#4lw.commands.whitelist=*
4lw.commands.whitelist=mntr,ruok
clientPort=2181
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/log
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=60
server.192=192.168.11.192:2888:3888
server.193=192.168.11.193:2888:3888
server.194=192.168.11.194:2888:3888
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
quorum.auth.enableSasl=true
quorum.auth.learnerRequireSasl=true
quorum.auth.serverRequireSasl=true
quorum.auth.learner.loginContext=QuorumLearner
quorum.auth.server.loginContext=QuorumServer
quorum.cnxn.threads.size=6
EOF
#zookeeper的sasl配置
cat > /data/zookeeper/conf/zk_server_jaas.conf << 'EOF'
Server {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin"
password="admin"
user_kafka="admin"
user_producer="admin";
};
QuorumServer {
org.apache.zookeeper.server.auth.DigestLoginModule required
user_zookeeper="zookeeper@password";
};
QuorumLearner {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="zookeeper"
password="zookeeper@password";
};
EOF
cat > /data/zookeeper/startup.sh << 'EOF'
docker run -d \
--network host \
--restart=always \
-e "SERVER_JVMFLAGS=-Djava.security.auth.login.config=/conf/zk_server_jaas.conf" \
-v /data/zookeeper/data:/data/zookeeper/data \
-v /data/zookeeper/conf:/conf \
-v /etc/localtime:/etc/localtime \
--name zookeeper \
zookeeper:3.6.3
EOF
bash /data/zookeeper/startup.sh
#查询zookeeper状态
docker exec -i zookeeper zkServer.sh status
zkui管理工具
mkdir -p /data/zkui/work
cat > /data/zkui/startup.sh << 'EOF'
docker run -d \
--name zkui \
--restart=always \
-e ZK_SERVER=192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 \
-v /etc/localtime:/etc/localtime \
-p 9090:9090 \
juris/zkui
EOF
bash /data/zkui/startup.sh
http://192.168.11.193:9090
用户名：admin/manager
配置kafka
所有节点
mkdir -p /data/kafka/{data,log,conf}
cat > /data/kafka/conf/kafka_server_jaas.conf << 'EOF'
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin"
password="admin"
user_admin="admin"
user_alice="alice";
};
Client {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="kafka"
password="admin";
};
EOF
#节点1
cat > /data/kafka/startup.sh << 'EOF'
docker run -d \
--name kafka \
--restart=always \
--network host \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_LISTENERS=SASL_PLAINTEXT://192.168.11.192:9092 \
-e KAFKA_ADVERTISED_LISTENERS=SASL_PLAINTEXT://192.168.11.192:9092 \
-e KAFKA_ZOOKEEPER_CONNECT=192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 \
-e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
-e KAFKA_OPTS=-Djava.security.auth.login.config=/data/kafka/conf/kafka_server_jaas.conf \
-e KAFKA_SECURITY_INTER_BROKER_PROTOCOL=SASL_PLAINTEXT \
-e KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN \
-e KAFKA_SASL_ENABLED_MECHANISMS=PLAIN \
-e KAFKA_PORT=9092 \
-e LOG_DIRS=/data/kafka/log \
-v /data/kafka/conf:/data/kafka/conf \
-v /data/kafka/log:/data/kafka/log \
-v /etc/localtime:/etc/localtime \
wurstmeister/kafka:latest
EOF
bash /data/kafka/startup.sh
节点2
cat > /data/kafka/startup.sh << 'EOF'
docker run -d \
--name kafka \
--restart=always \
--network host \
-e KAFKA_BROKER_ID=2 \
-e KAFKA_LISTENERS=SASL_PLAINTEXT://192.168.11.193:9092 \
-e KAFKA_ADVERTISED_LISTENERS=SASL_PLAINTEXT://192.168.11.193:9092 \
-e KAFKA_ZOOKEEPER_CONNECT=192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 \
-e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
-e KAFKA_OPTS=-Djava.security.auth.login.config=/data/kafka/conf/kafka_server_jaas.conf \
-e KAFKA_SECURITY_INTER_BROKER_PROTOCOL=SASL_PLAINTEXT \
-e KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN \
-e KAFKA_SASL_ENABLED_MECHANISMS=PLAIN \
-e KAFKA_PORT=9092 \
-e LOG_DIRS=/data/kafka/log \
-v /data/kafka/conf:/data/kafka/conf \
-v /data/kafka/log:/data/kafka/log \
-v /etc/localtime:/etc/localtime \
wurstmeister/kafka:latest
EOF
bash /data/kafka/startup.sh
节点3
cat > /data/kafka/startup.sh << 'EOF'
docker run -d \
--name kafka \
--restart=always \
--network host \
-e KAFKA_BROKER_ID=3 \
-e KAFKA_LISTENERS=SASL_PLAINTEXT://192.168.11.194:9092 \
-e KAFKA_ADVERTISED_LISTENERS=SASL_PLAINTEXT://192.168.11.194:9092 \
-e KAFKA_ZOOKEEPER_CONNECT=192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 \
-e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
-e KAFKA_OPTS=-Djava.security.auth.login.config=/data/kafka/conf/kafka_server_jaas.conf \
-e KAFKA_SECURITY_INTER_BROKER_PROTOCOL=SASL_PLAINTEXT \
-e KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN \
-e KAFKA_SASL_ENABLED_MECHANISMS=PLAIN \
-e KAFKA_PORT=9092 \
-e LOG_DIRS=/data/kafka/log \
-v /data/kafka/conf:/data/kafka/conf \
-v /data/kafka/log:/data/kafka/log \
-v /etc/localtime:/etc/localtime \
wurstmeister/kafka:latest
EOF
bash /data/kafka/startup.sh
- 确认kafka集群状态
- 建新的topic
docker exec -it kafka \
kafka-topics.sh --create --zookeeper 192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 --topic test --partitions 3 --replication-factor 1
Created topic test.
- 向主题中写入内容
#进入容器
docker exec -it kafka bash
#创建sasl验证配置文件
cat > /data/kafka/conf/kafka_client_jaas.conf << 'EOF'
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin"
password="admin";
};
Client {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin"
password="admin";
};
EOF
#修改consumer.properties和producer.properties
echo -e 'security.protocol=SASL_PLAINTEXT\nsasl.mechanism=PLAIN' >> /opt/kafka_2.13-2.7.0/config/consumer.properties
echo -e 'security.protocol=SASL_PLAINTEXT\nsasl.mechanism=PLAIN' >> /opt/kafka_2.13-2.7.0/config/producer.properties
kafka-console-producer.sh --broker-list 192.168.11.192:9092,192.168.11.193:9092,192.168.11.194:9092 --topic test --producer.config /opt/kafka_2.13-2.7.0/config/producer.properties
##消费内容
#创建sasl验证配置文件
cat > /data/kafka/conf/kafka_client_jaas.conf << 'EOF'
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin"
password="admin";
};
Client {
org.apache.zookeeper.server.auth.DigestLoginModule required
username="admin"
password="admin";
};
EOF
#修改consumer.properties和producer.properties
echo -e 'security.protocol=SASL_PLAINTEXT\nsasl.mechanism=PLAIN' >> /opt/kafka_2.13-2.7.0/config/consumer.properties
echo -e 'security.protocol=SASL_PLAINTEXT\nsasl.mechanism=PLAIN' >> /opt/kafka_2.13-2.7.0/config/producer.properties
export KAFKA_OPTS=" -Djava.security.auth.login.config=/data/kafka/conf/kafka_client_jaas.conf"
#消费消息
kafka-console-consumer.sh --bootstrap-server 192.168.11.193:9092 --topic test --from-beginning --consumer.config /opt/kafka_2.13-2.7.0/config/consumer.properties
生产者：
消费者:
kafka_manager
mkdir -p /data/kafka_manager
cat > /data/kafka_manager/startup.sh << 'EOF'
docker run -d \
--name kafka-manager \
--restart=always \
-e ZK_HOSTS=192.168.11.192:2181,192.168.11.193:2181,192.168.11.194:2181 \
-e KAFKA_MANAGER_AUTH_ENABLED=true \
-e KAFKA_MANAGER_USERNAME=admin \
-e KAFKA_MANAGER_PASSWORD=admin \
-v /etc/localtime:/etc/localtime \
-p 9000:9000 \
kafkamanager/kafka-manager
EOF
bash /data/kafka_manager/startup.sh
http://192.168.11.193:9000
user: admin
password: admin
kafka_manager介面配置（文本）
cluster name---
admin
cluster zookeeper hosts---
192.168.11.211:2181,192.168.11.212:2181,192.168.11.213:2181/
sasl jaas config-----
org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin" ;
其它：
Kafka全网最全最详细运维命令合集 https://blog.csdn.net/u010634066/article/details/118215928
https://blog.csdn.net/easylife206/article/details/112645749