k8s 日志收集、es+ Filebeat +kibala

系列文章目录

实现原理
一个java副本程序里包含一个 filebeat 程序，java程序固定输入到目录日志，filebeat 进行采集，推送到 es，kibala 进行采集

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: $IMAGES
  name: $IMAGES
  namespace: yanxuan
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: $IMAGES
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 25%
      maxSurge: 25%
  template:
    metadata:
      labels:
        app: $IMAGES
    spec:
      imagePullSecrets:
        - name: aliyun-docker-hub
      containers:
          
        - image: 'elastic/filebeat:7.17.3' 
          args: [
            "-c","/etc/filebeat.yml",
            "-e",
          ]  
          name: filebeat
          resources:
            limits:
              cpu: '500'
              memory: 500Mi
            requests:
              cpu: '0.1'
              memory: 100Mi 
          volumeMounts: 
            - name: $IMAGES
              mountPath: /data/logs           #挂载收集日志目录
            - name: filebeat-config           #挂载 filebeat ConfigMap的日志收集脚本
              mountPath: /etc/filebeat.yml     
              subPath: filebeat.yml           #指定子路径
              
        - image: '$REGISTRY/$DOCKERHUB_NAMESPACE/$IMAGES:$BUILD_NUMBER'  #后端java 服务
          name: $IMAGES
          ports:
            - containerPort: $JAR_PORD
              protocol: TCP
          resources:            #这里是最大使用
            limits:
              cpu: '2.5'
              memory: 2500Mi
            requests:          #这里是最小使用
              cpu: '0.1'
              memory: 256Mi
              
          volumeMounts:          #将java日志输出到 挂载目录下/data/logs 
            - name: $IMAGES
              mountPath: /data/logs 
              
          livenessProbe:             #健康检查
            initialDelaySeconds: 60
            timeoutSeconds: 10
            periodSeconds: 10
            successThreshold: 1
            failureThreshold: 5
            httpGet:
              scheme: HTTP
              path: $INSPECT
              port: $JAR_PORD
              
      volumes:
        - name: $IMAGES
          emptyDir: {}
        - name: filebeat-config
          configMap:
            name: filebeat-config        
      terminationGracePeriodSeconds: 30
---
apiVersion: v1
kind: ConfigMap          
metadata:
  namespace: yanxuan
  name: filebeat-config
data:
  filebeat.yml: |-                #配置字典、日志采集格式  
    filebeat.inputs:
    - type: log
      paths:
        - /data/logs/console.log        #获取的logs
      fields:
        type: "$IMAGES"
      multiline:
        pattern: '^202.*'   #分割 以时间开头分割
        negate: true
        match: after

    setup.ilm.enabled: false
    setup.template.name: "log"
    setup.template.pattern: "log-*"

    output.elasticsearch:
      hosts: ["$ES:9200"]
      indices:
        - index: "$IMAGES-%{+yyyy.MM.dd}"
          when.equals:
            fields.type: "$IMAGES"

    setup.kibana:
      host: "$ES:5601"
    processors:
      - script:
          lang: javascript
          id: my_filter
          tag: enable
          source: >
            function process(event) {
                var message= event.Get("message");
                var time =message.split("|")[0];
                event.Put("log_time",time);
                message= message.replace(time, '')
                event.Put("message", message);      

                var message= event.Get("message");
                var time =message.split("|")[3];
                event.Put("ERROR",time);
              
            }
      - timestamp:
          field: log_time
          timezone: Asia/Shanghai
          layouts:
            - '2006-01-02 15:04:05'
            - '2006-01-02 15:04:05.999'
          test:
            - '2019-06-22 16:33:51'