k8s部署elk+filebeat+logstash+kafka集群（二）kafka-kraft模式集群

​

1、构建kafka镜像

kafka没有找到官方镜像，所以自己构建个kafka镜像，我使用的版本是2.8
下载地址：https://kafka.apache.org/

# 编辑启动脚本 cat start.sh
# !/bin/bash
UIDPATH=/kafka/uuid/uuid.txt
if [ ! -f $UIDPATH ]
then
        kafka-storage.sh random-uuid > /kafka/uuid/uuid.txt
fi
randomid=`cat /kafka/uuid/uuid.txt`
kafka-storage.sh format -t $randomid -c $KAFKA_HOME/config/kraft/server.properties
kafka-server-start.sh $KAFKA_HOME/config/kraft/server.properties

# 编写Dockerfile cat Dockerfile
FROM debian:stable-slim
MAINTAINER [email protected]
LABEL version="2.8.0"

ENV KAFKA_HOME=/kafka
ENV PATH=$PATH:$KAFKA_HOME/bin
ENV JAVA_HOME=/jdk1.8.0_281
ENV JRE_HOME=$JAVA_HOME/jre
ENV CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JRE_HOME/lib
ENV PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

ADD kafka_2.12-2.8.0.tgz /
ADD jdk-8u281-linux-x64.tar.gz /
ADD start.sh /kafka_2.12-2.8.0
RUN rm -rf /jdk1.8.0_281/*.zip \
    && mkdir -p /kafka_2.12-2.8.0/uuid \
    && chmod 755 -R /kafka_2.12-2.8.0 \
    && mv /kafka_2.12-2.8.0 /kafka \
    && groupadd -g 1000 kafka && useradd -u 1000 kafka -g 1000 -d /kafka \
    && chown  -R 1000:1000 /kafka

WORKDIR $KAFKA_HOME

EXPOSE 9092 9093

USER 1000:1000

CMD ["/bin/bash","-c","$KAFKA_HOME/start.sh"]

将start.sh,Dockerfile,以及jdk包，kafka安装包都放在同一个目录下，开始构建镜像
docker build -f ./Dockerfile -t precomp/kafka:2.8.0
好了，镜像构建完毕就可以上传到我们的镜像仓库中开始部署了

2、部署kafka集群并做持久化存储

持久化

# cat kafka-dep-storageclass.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-nfs-client-provisioner
  labels:
    app: kafka-nfs-client-provisioner
  namespace: default
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: kafka-nfs-client-provisioner
  template:
    metadata:
      labels:
        app: kafka-nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: kafka-nfs-client-provisioner
          image: 3.127.33.174:8443/kubernetes/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: kafka/ifs
            - name: NFS_SERVER
              value: 3.127.10.233
            - name: NFS_PATH
              value: /home/k8s/kafka/logs
      volumes:
        - name: nfs-client-root
          nfs:
            server: 3.127.10.233
            path: /home/k8s/kafka/logs
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: kafka-managed-nfs-storage
provisioner: kafka/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "true"
reclaimPolicy: Retain

rbac鉴权

# cat rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: kafka
  namespace: elk
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kafka
  labels:
    app: kafka-clusterrole
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - events
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
- apiGroups:
  - ""
  resourceNames:
  - kafka-prospectors
  resources:
  - configmaps
  verbs:
  - get
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kafka
  labels:
    app: kafka-clusterrolebinding
roleRef:
  apiGroup: ""
  kind: ClusterRole
  name: kafka
subjects:
- apiGroup: ""
  kind: ServiceAccount
  name: kafka
  namespace: elk

​kafka集群

# cat kafka-kraft-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: kafka-configmap
  namespace: elk
data:
  start-kafka.sh: |
    #!/bin/bash
    echo 44m6ypTXR4iJCJ9Boc9MbQ > /kafka/uuid/uuid.txt
    cat>/kafka/config/kraft/server.properties<<EOF
    process.roles=broker,controller
    node.id=${HOSTNAME##*-}
    controller.quorum.voters=0@kafka-kraft-statefulset-0.kafka-kraft-svc:9093,[email protected]:9093,[email protected]:9093
	# 这里kafka启用了两个监听器，k8s集群内直接使用INSIDE访问，因为podip会变而域名不会改变所以重启kafka的时候也不会导致logstash连接不上
    listeners=INSIDE://0.0.0.0:9091,CONTROLLER://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
    inter.broker.listener.name=INSIDE
    advertised.listeners=INSIDE://${HOSTNAME}.kafka-kraft-svc:9091,OUTSIDE://${HOST_IP}:9092
    #advertised.listeners=PLAINTEXT://${HOST_IP}:9092
    controller.listener.names=CONTROLLER
    listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
    num.network.threads=9
    num.io.threads=9
    #socket.send.buffer.bytes=102400
    #socket.receive.buffer.bytes=102400
    #socket.request.max.bytes=1048576000
    log.dirs=/kafka/kraft-combined-logs
    num.partitions=3
    num.recovery.threads.per.data.dir=1
    offsets.topic.replication.factor=3
    transaction.state.log.replication.factor=3
    transaction.state.log.min.isr=1
    log.retention.hours=48
    #log.segment.bytes=1073741824
    #log.retention.check.interval.ms=300000
    delete.topic.enable=true
    host.name=${HOST_IP}
    EOF
    /kafka/bin/kafka-storage.sh format -t `cat /kafka/uuid/uuid.txt` -c $KAFKA_HOME/config/kraft/server.properties
    /kafka/bin/kafka-server-start.sh $KAFKA_HOME/config/kraft/server.properties

# cat kafka-kraft-statefulset.yaml

apiVersion: v1
kind: Service
metadata:
  name: kafka-kraft-svc
  namespace: elk
  labels:
    app: kafka-cluster-svc
spec:
  selector:
    app: kafka-kraft-2.8.0
  type: ClusterIP
  clusterIP: None
  #publishNotReadyAddresses: true
  sessionAffinity: ClientIP
  ports:
  - name: outer-port
    port: 9092
    protocol: TCP
    targetPort: 9092
  - name: cluster-port
    port: 9093
    protocol: TCP
    targetPort: 9093
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka-kraft-statefulset
  namespace: elk
spec:
  podManagementPolicy: OrderedReady
  replicas: 3
  selector:
    matchLabels:
      app: kafka-kraft-2.8.0
  serviceName: kafka-kraft-svc
  template:
    metadata:
      name: kafka-kraft
      namespace: elk
      labels:
        app: kafka-kraft-2.8.0
    spec:
      serviceAccountName: kafka
      containers:
      - name: kafka-kraft
        image: 3.127.33.174:8443/elk/kafka:2.8.0
        imagePullPolicy: Always
        resources:
          limits:
            memory: "4Gi"
            cpu: "512m"
        env:
        - name: NODE_ID
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: KAFKA_HEAP_OPTS
          value: -Xmx512M -Xms512M
        - name: HOST_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: TZ
          value: Asia/Shanghai
        command: ["/bin/bash","-c"]
        args: ["/kafka/start-kafka.sh"]
        ports:
        - name: interactiveport
          containerPort: 9092
          protocol: TCP
        - name: controlport
          containerPort: 9093
          protocol: TCP
        volumeMounts:
        - name: kafka-cluster-claim
          mountPath: /kafka/kraft-combined-logs/
        - name: config
          mountPath: /kafka/start-kafka.sh
          subPath: start-kafka.sh
        - name: timezone
          mountPath: /etc/localtime
        livenessProbe:
          periodSeconds: 15
          tcpSocket:
            port: 9092
      volumes:
      - name: config
        configMap:
          defaultMode: 493
          name: kafka-configmap
      - name: timezone
        hostPath:
          path: /etc/localtime
      tolerations:
      - key: node-role.kubernetes.io
        effect: NoSchedule
        operator: Equal
        value: master
      - effect: NoSchedule
        operator: Exists
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: node
                operator: In
                values:
                - master1
                - node1
  volumeClaimTemplates:
  - metadata:
      name: kafka-cluster-claim
      namespace: elk
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        limits:
          storage: 60Gi
        requests:
          storage: 40Gi
      storageClassName: kafka-managed-nfs-storage
  updateStrategy:
    type: RollingUpdate

svc

# cat kafka-cluster-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: kafka-0
  namespace: elk
spec:
  type: NodePort
  clusterIP: 10.99.100.100
  ports:
  - name: kafkaport
    port: 9092
    targetPort: 9092
    nodePort: 30090
    protocol: TCP
  selector:
    statefulset.kubernetes.io/pod-name: kafka-kraft-statefulset-0
---
apiVersion: v1
kind: Service
metadata:
  name: kafka-1
  namespace: elk
spec:
  type: NodePort
  clusterIP: 10.99.100.101
  ports:
  - name: kafkaport
    port: 9092
    targetPort: 9092
    nodePort: 30091
    protocol: TCP
  selector:
    statefulset.kubernetes.io/pod-name: kafka-kraft-statefulset-1
---
apiVersion: v1
kind: Service
metadata:
  name: kafka-2
  namespace: elk
spec:
  type: NodePort
  clusterIP: 10.99.100.102
  ports:
  - name: kafkaport
    port: 9092
    targetPort: 9092
    nodePort: 30092
    protocol: TCP
  selector:
    statefulset.kubernetes.io/pod-name: kafka-kraft-statefulset-2

至此kafka集群部署完毕，可以进入容器操作kafka查看是否成功运行
./bin/kafka-topic.sh --bootstrap-server localhost:9091 --create --topic test
./bin/kafka-topic.sh --bootstrap-server localhost:9092 --list

最后我遇到了一些问题，我创建三个svc实际上是为了对外暴露服务，让k8s集群外的filebeat可以成功推送数据到kafka中，我甚至在集群外部也安装了kafka以便使用客户端查看是否能连接3.127.10.209:30090端口消费到k8s中kafka的数据，开始advertised.listeners我使用的是svc的三个地址，就是10.99那三个svc，kafka可以收到k8s集群外部的filebeat推送的数据，只是不知什么原因数据会有遗漏，但是隔天之后k8s集群外部的kafka客户端或者filebeat就无法再连接kafka，研究了好几天实在没有发现问题出在哪里，中间并没有修改过配置，突然再也无法连接了，如果不使用kafka直接将数据推送到logstash中，没有发现遗漏情况而且实时性也很高