k8s部署elk+filebeat+logstash+kafka集群（二）kafka-zookeeper模式集群
zookeeper集群部署
zookeeper使用官方镜像，版本为3.5，上传到镜像仓库
rbac
apiVersion: v1
kind: ServiceAccount
metadata:
name: kafka
namespace: elk
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kafka
labels:
app: kafka-clusterrole
rules:
- apiGroups:
- ""
resources:
- nodes
- events
- namespaces
- pods
verbs:
- get
- watch
- list
- apiGroups:
- ""
resourceNames:
- kafka-prospectors
resources:
- configmaps
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kafka
labels:
app: kafka-clusterrolebinding
roleRef:
apiGroup: ""
kind: ClusterRole
name: kafka
subjects:
- apiGroup: ""
kind: ServiceAccount
name: kafka
namespace: elk
zk集群
# cat zk-statefulset.yaml
apiVersion: v1
kind: Service
metadata:
name: zk-svc
namespace: elk
labels:
app: zk-cluster-svc
spec:
selector:
app: zookeeper-3.5
type: ClusterIP
clusterIP: None
#publishNotReadyAddresses: true
sessionAffinity: ClientIP
ports:
- name: clientport
port: 2181
protocol: TCP
targetPort: 2181
- name: select-port
port: 3888
protocol: TCP
targetPort: 3888
- name: leader-port
port: 2888
protocol: TCP
targetPort: 2888
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: zk-statefulset
namespace: elk
spec:
podManagementPolicy: OrderedReady
replicas: 3
selector:
matchLabels:
app: zookeeper-3.5
serviceName: zk-svc
template:
metadata:
name: zk
namespace: elk
labels:
app: zookeeper-3.5
spec:
serviceAccountName: kafka
containers:
- name: kafka
image: 3.127.33.174:8443/elk/zookeeper:3.5
imagePullPolicy: IfNotPresent
resources:
limits:
memory: "1Gi"
cpu: "256m"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: ZOO_SERVERS
value: server.0=zk-statefulset-0.zk-svc:2888:3888;2181 server.1=zk-statefulset-1.zk-svc:2888:3888;2181 server.2=zk-statefulset-2.zk-svc:2888:3888;2181
- name: ZOO_TICK_TIME
value: "10000"
- name: ZOO_DATA_LOG_DIR
value: /data/log
args: ["/bin/sh","-c","echo 'clientPort=2181' >> /conf/zoo.cfg; echo ${HOSTNAME##*-} > /data/myid; ./bin/zkServer.sh start-foreground"]
ports:
- name: clientport
containerPort: 2181
protocol: TCP
- name: leaderport
containerPort: 2888
protocol: TCP
- name: select-leader
containerPort: 3888
protocol: TCP
volumeMounts:
- name: zk-cluster-claim
mountPath: /data
- name: timezone
mountPath: /etc/localtime
livenessProbe:
periodSeconds: 30
tcpSocket:
port: 2181
volumes:
- name: timezone
hostPath:
path: /etc/localtime
tolerations:
- key: node-role.kubernetes.io
effect: NoSchedule
operator: Equal
value: master
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: node
operator: In
values:
- master1
- node1
volumeClaimTemplates:
- metadata:
name: zk-cluster-claim
namespace: elk
spec:
accessModes:
- ReadWriteMany
resources:
limits:
storage: 5Gi
storageClassName: zk-managed-nfs-storage
updateStrategy:
type: RollingUpdate
zk集群部署完毕
kafka集群部署
镜像依旧使用之前kraft模式构建的kafka镜像
# cat kafka-zk-statefulset.yaml
apiVersion: v1
kind: Service
metadata:
name: kafka-zk-svc
namespace: elk
labels:
app: kafka-cluster-svc
spec:
selector:
app: kafka-zk-2.8.0
type: ClusterIP
clusterIP: None
#publishNotReadyAddresses: true
sessionAffinity: ClientIP
ports:
- name: outer-port
port: 9092
protocol: TCP
targetPort: 9092
- name: cluster-port
port: 9093
protocol: TCP
targetPort: 9093
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: kafka-zk-statefulset
namespace: elk
spec:
podManagementPolicy: OrderedReady
serviceName: kafka
replicas: 3
selector:
matchLabels:
app: kafka-zk-2.8.0
serviceName: kafka-zk-svc
template:
metadata:
name: kafka-zk
namespace: elk
labels:
app: kafka-zk-2.8.0
spec:
serviceAccountName: kafka
containers:
- name: kafka-zk
image: 3.127.33.174:8443/elk/kafka:2.8.0
imagePullPolicy: IfNotPresent
resources:
limits:
memory: "4Gi"
cpu: "512m"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: KAFKA_HEAP_OPTS
value: -Xmx1G -Xms1G
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
command: ["/bin/bash","-c"]
args: ["/kafka/start-kafka.sh"]
ports:
- name: interactiveport
containerPort: 9092
protocol: TCP
- name: controlport
containerPort: 9093
protocol: TCP
volumeMounts:
- name: kafka-cluster-claim
mountPath: /kafka/kafka-zk-logs/
- name: config
mountPath: /kafka/start-kafka.sh
subPath: start-kafka.sh
- name: timezone
mountPath: /etc/localtime
livenessProbe:
periodSeconds: 15
tcpSocket:
port: 9092
volumes:
- name: config
configMap:
defaultMode: 493
name: kafka-zk-configmap
- name: timezone
hostPath:
path: /etc/localtime
tolerations:
- key: node-role.kubernetes.io
effect: NoSchedule
operator: Equal
value: master
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- key: node
operator: In
values:
- master1
- node1
volumeClaimTemplates:
- metadata:
name: kafka-cluster-claim
namespace: elk
spec:
accessModes:
- ReadWriteMany
resources:
limits:
storage: 60Gi
requests:
storage: 40Gi
storageClassName: kafka-zk-managed-nfs-storage
updateStrategy:
type: RollingUpdate
查看运行状态kubectl get pods -n elk | grep zk
进入kafka执行相关操作进行验证
kafka常用主题命令
查看操作主题命令参数
--bootstrap-server <String: server toconnect to>
连接kafka broker主机名称和端口号
--topic <String: topic>
操作的topic名称
--create
创建主题
--delete
删除主题
--alter
修改主题
--list
查看所有主题
--describe
查看主题详细描述
--partitions <Integer: # of partitions>
设置分区数
--replication-factor <Integer: replication factor>
设置分区副本
--config <String: name=value>