k8s部署rocketmq单节点（server+broker+console）—— 筑梦之路
之前写过几篇关于rocketmq构建双架构镜像、docker-compose部署的文章，接上篇docker-compose部署rocketmq，这里记录下k8s下如何部署。
多架构环境下docker-compose部署rocketmq单机模式—— 筑梦之路-CSDN博客
nameserver节点
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: rocketmq-nameserv
labels: {app: rocketmq}
namespace: merry
spec:
serviceName: rocketmq
replicas: 1
selector:
matchLabels: {app: rocketmq}
template:
metadata:
labels: {app: rocketmq}
spec:
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: appnode
operator: In
values:
- true
containers:
- name: rocketmq
image: harbor.codemiracle.com.cn/baseapp/rocketmq:4.7.1
imagePullPolicy: IfNotPresent
command: ["/home/rocketmq/rocketmq-4.7.1/bin/mqnamesrv"]
resources:
limits: {cpu: 5000m, memory: 2000Mi}
requests: {cpu: 1000m, memory: 2000Mi}
env:
- {name: TZ, value: Asia/Shanghai}
#- {name: JAVA_OPT, value: -Duser.home=/opt}
#- {name: JAVA_OPT_EXT, value: -server -Xms1900m -Xmx1900m -Xmn1024m}
ports:
- {containerPort: 9876}
volumeMounts:
- {name: vol-init-d, mountPath: /home/rocketmq/logs}
volumeClaimTemplates:
- metadata: {name: vol-init-d}
spec:
accessModes: [ReadWriteOnce]
resources:
requests: {storage: 1000Gi}
storageClassName: custom-hostpath
---
apiVersion: v1
kind: Service
metadata:
name: rocketmq-svc
labels:
app: rocketmq
namespace: merry
spec:
type: NodePort
ports:
- port: 9876
targetPort: 9876
nodePort: 9876
selector:
app: rocketmqbroker节点
---
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-config
namespace: merry
data:
broker.conf: |
#所属集群名字
brokerClusterName=DefaultCluster
#broker名字，注意此处不同的配置文件填写的不一样，如果在broker-a.properties使用:broker-a,
#在broker-b.properties使用:broker-b
brokerName=broker-a
#0 表示Master，>0 表示Slave
brokerId=0
#nameServer地址，分号分割
#namesrvAddr=rocketmq-0:9876;rocketmq-1:9876
namesrvAddr=rocketmq-svc:9876
#启动IP,如果 docker 报 com.alibaba.rocketmq.remoting.exception.RemotingConnectException: connect to <192.168.0.120:10909> failed
# 解决方式1 加上一句producer.setVipChannelEnabled(false);，解决方式2 brokerIP1 设置宿主机IP， 不要使用docker 内部IP
#brokerIP1=192.168.0.254
brokerIP1=192.168.100.100
#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数
defaultTopicQueueNums=4
#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭 ！！！这里仔细看是false，false，false
#原因下篇博客见~ 哈哈哈哈
autoCreateTopicEnable=true
#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭
autoCreateSubscriptionGroup=true
#Broker 对外服务的监听端口
listenPort=10911
#haService中使用 默认值为：listenPort + 1
haListenPort=10912
#主要用于slave同步master listenPort - 2
fastListenPort=10909
#删除文件时间点，默认凌晨4点
deleteWhen=04
#文件保留时间，默认48小时
fileReservedTime=48
#commitLog每个文件的大小默认1G
mapedFileSizeCommitLog=1073741824
#ConsumeQueue每个文件默认存30W条，根据业务情况调整
mapedFileSizeConsumeQueue=300000
#destroyMapedFileIntervalForcibly=120000
#redeleteHangedFileInterval=120000
#检测物理文件磁盘空间
diskMaxUsedSpaceRatio=88
#存储路径
storePathRootDir=/home/rocketmq/store
#commitLog 存储路径
#storePathCommitLog=/home/ztztdata/rocketmq-all-4.1.0-incubating/store/commitlog
#消费队列存储
#storePathConsumeQueue=/home/ztztdata/rocketmq-all-4.1.0-incubating/store/consumequeue
#消息索引存储路径
#storePathIndex=/home/ztztdata/rocketmq-all-4.1.0-incubating/store/index
#checkpoint 文件存储路径
#storeCheckpoint=/home/ztztdata/rocketmq-all-4.1.0-incubating/store/checkpoint
#abort 文件存储路径
#abortFile=/home/ztztdata/rocketmq-all-4.1.0-incubating/store/abort
#限制的消息大小
maxMessageSize=4194304
#flushCommitLogLeastPages=4
#flushConsumeQueueLeastPages=2
#flushCommitLogThoroughInterval=10000
#flushConsumeQueueThoroughInterval=60000
#Broker 的角色
#- ASYNC_MASTER 异步复制Master
#- SYNC_MASTER 同步双写Master
#- SLAVE
brokerRole=ASYNC_MASTER
#刷盘方式
#- ASYNC_FLUSH 异步刷盘
#- SYNC_FLUSH 同步刷盘
flushDiskType=ASYNC_FLUSH
#发消息线程池数量
sendMessageThreadPoolNums=64
#拉消息线程池数量
pullMessageThreadPoolNums=64
#消息发送队列等待时间，默认200
waitTimeMillsInSendQueue=400
#发送消息是否使用可重入锁
useReentrantLockWhenPutMessage=true
---
apiVersion: v1
kind: Service
metadata:
name: rmqbroker
labels:
app: rmqbroker
namespace: merry
spec:
type: NodePort
ports:
- port: 10909
targetPort: 10909
name: slave-master
nodePort: 10909
- port: 10911
targetPort: 10911
name: listen
nodePort: 10911
- port: 10912
targetPort: 10912
name: ha-service
nodePort: 10912
selector:
app: rmqbroker
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: rmqbroker
labels: {app: rmqbroker}
namespace: merry
spec:
serviceName: rmqbroker
replicas: 1
selector:
matchLabels: {app: rmqbroker}
template:
metadata:
labels: {app: rmqbroker}
spec:
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: rmbroker0node
operator: In
values:
- true
containers:
- name: rmqbroker
image: harbor.codemiracle.com.cn/baseapp/rocketmq:4.7.1
imagePullPolicy: IfNotPresent
command: ["/bin/sh","-c","/home/rocketmq/rocketmq-4.7.1/bin/mqbroker -c /etc/rocketmq/broker.conf"]
resources:
limits: {cpu: 4000m, memory: 8Gi}
requests: {cpu: 1000m, memory: 2Gi}
env:
- {name: TZ, value: Asia/Shanghai}
- {name: JAVA_OPT_EXT, value: -Duser.home=/opt -server -Xms1536m -Xmx4096m}
#command: [/bin/bash, mqbroker, -c, /etc/rocketmq/broker.conf, -n, 'rocketmq-svc:9876',
# autoCreateTopicEnable=true]
ports:
- {containerPort: 10909}
- {containerPort: 10911}
- {containerPort: 10912}
volumeMounts:
- {name: vol-store, mountPath: /home/rocketmq/store}
- {name: vol-broker-config, mountPath: /etc/rocketmq}
volumes:
- name: vol-broker-config
configMap: {name: broker-config}
volumeClaimTemplates:
- metadata: {name: vol-store}
spec:
accessModes: [ReadWriteOnce]
resources:
requests: {storage: 1000Gi}
storageClassName: custom-hostpath
console客户端
kind: ConfigMap
apiVersion: v1
metadata:
name: rocketmq-console-cm
namespace: merry
data:
users.properties: |+
# 该文件支持热修改，即添加和修改用户时，不需要重新启动console
# 格式， 每行定义一个用户， username=password[,N] #N是可选项，可以为0 (普通用户)； 1 （管理员）
#定义管理员
admin=admin123,1
#定义普通用户
#user1=user1
#user2=user2
role-permission.yml: |+
# 该文件支持热修改，即添加和修改用户时，不需要重新启动console
# 格式，如果增加和删除接口权限，直接在列表中增加和删除接口地址即可。
# 接口路径配置支持通配符
# * 表示匹配0或多个不是/的字符
# ** 表示匹配0或多个任意字符
# ? 表示匹配1个任意字符
rolePerms:
# 普通用户
ordinary:
- /rocketmq/nsaddr
- /ops/*
- /dashboard/**
- /topic/*.query
- /topic/sendTopicMessage.do
- /producer/*.query
- /message/*
- /messageTrace/*
- /monitor/*
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
run: rocketmq-console
name: rocketmq-console-deploy
namespace: merry
spec:
replicas: 1
selector:
matchLabels:
run: rocketmq-console
template:
metadata:
labels:
run: rocketmq-console
spec:
containers:
- name: rocketmq-console
image: harbor.codemiracle.com.cn/baseapp/rocketmq-console:2.0.0
env:
- name: TZ
value: Aisa/Shanghai
- name: JAVA_OPTS
value: "-Drocketmq.namesrv.addr=rocketmq-svc:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false -Drocketmq.config.loginRequired=true -Drocketmq.config.dataPath=/tmp/rocketmq-console/data"
volumeMounts:
- mountPath: /tmp/rocketmq-console/data/users.properties
name: rocketmq-console-cm
subPath: users.properties
- mountPath: /tmp/rocketmq-console/data/role-permission.yml
name: rocketmq-console-cm
subPath: role-permission.yml
volumes:
- name: rocketmq-console-cm
configMap:
name: rocketmq-console-cm
---
apiVersion: v1
kind: Service
metadata:
labels:
run: rocketmq-console
name: rocketmq-console
namespace: merry
spec:
type: NodePort
ports:
- name: http
protocol: TCP
targetPort: 8080
port: 8080
nodePort: 18080
selector:
run: rocketmq-console这里主要以rocketmq 4.7.1为例，其他版本类似。
-----------------------------------------------------------------------
FAQ
2023年12月4日 经测试，以上部署会存在无法写入和消费的问题，使用docker-compose部署正常，测试过程如下：
# 进入broker节点容器内
# 发送消息
export NAMESRV_ADDR=rocketmq-svc:9876
sh bin/tools.sh org.apache.rocketmq.example.quickstart.Producer
# 接收消息
sh bin/tools.sh org.apache.rocketmq.example.quickstart.Consumer
# 如果能正常生产 消费则正常，反之则反。如何解决该问题？
更换镜像为king019/rocketmq:4.8.0
docker hub地址：https://hub.docker.com/r/king019/rocketmq
修改deploy yaml 中的启动命令
# nameserver
command: ["/opt/soft/rocketmq/bin/mqnamesrv"]
# broker
command: ["/bin/sh","-c","/opt/soft/rocketmq/bin/mqbroker -c /etc/rocketmq/broker.conf"]添加环境变量
- {name: JAVA_HOME, value: /usr/lib/jvm/java-1.8-openjdk}修改挂载日志路径
/root/logs/rocketmqlogs/然后再次测试，均能正常使用。