fluent-bit对接loki收集日志

Loki搭建

指定端口3100，以及配置文件即可，如下图所示：

使用镜像：grafana/loki:v1.0.0
loki配置文件如下：

auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 1

schema_config:
  configs:
  - from: 2018-04-15
    store: boltdb
    object_store: filesystem
    schema: v9
    index:
      prefix: index_
      period: 168h

storage_config:
  boltdb:
    directory: /tmp/loki/index

  filesystem:
    directory: /tmp/loki/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0

table_manager:
  chunk_tables_provisioning:
    inactive_read_throughput: 0
    inactive_write_throughput: 0
    provisioned_read_throughput: 0
    provisioned_write_throughput: 0
  index_tables_provisioning:
    inactive_read_throughput: 0
    inactive_write_throughput: 0
    provisioned_read_throughput: 0
    provisioned_write_throughput: 0
  retention_deletes_enabled: false
  retention_period: 0

fluent-bit搭建

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit
  namespace: monitor
  labels:
    app: fluent-bit-loki
    chart: fluent-bit-0.0.2
    release: romping-tapir
    heritage: Tiller
data:
  fluent-bit.conf: |-
    [SERVICE]
        Flush          1
        Daemon         Off
        Log_Level      warn
        Parsers_File   parsers.conf
    [INPUT]
        Name           tail
        Tag            kube.*
        Path           /var/log/containers/*.log
        Parser         docker
        DB             /run/fluent-bit/flb_kube.db
        Mem_Buf_Limit  5MB
    [FILTER]
        Name           kubernetes
        Match          kube.*
        Kube_URL       https://kubernetes.default.svc:443
        Merge_Log On
    [Output]
        Name loki
        Match *
        Url http://loki-test.monitor:3100/loki/api/v1/push     #loki服务地址
        Labels {job="fluent-bit"}
        RemoveKeys kubernetes,stream
        LabelMapPath /fluent-bit/etc/labelmap.json
        LineFormat json
        LogLevel warn

  parsers.conf: |-
    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

  labelmap.json: |-
    {
      "kubernetes": {
        "container_name": "container",
        "host": "node",
        "labels": {
          "app": "app",
          "release": "release"
        },
        "namespace_name": "namespace",
        "pod_name": "instance"
      },
      "stream": "stream"
    }
---
# Source: fluent-bit/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: fluent-bit-loki
    chart: fluent-bit-0.0.2
    heritage: Tiller
    release: romping-tapir
  name: fluent-bit
  namespace: monitor
---
# Source: fluent-bit/templates/clusterrole.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    app: fluent-bit-loki
    chart: fluent-bit-0.0.2
    release: romping-tapir
    heritage: Tiller
  name: fluent-bit-clusterrole
  namespace: monitor
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs: ["get", "watch", "list"]
---
# Source: fluent-bit/templates/clusterrolebinding.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluent-bit-clusterrolebinding
  labels:
    app: fluent-bit-loki
    chart: fluent-bit-0.0.2
    release: romping-tapir
    heritage: Tiller
subjects:
  - kind: ServiceAccount
    name: fluent-bit
    namespace: monitor
roleRef:
  kind: ClusterRole
  name: fluent-bit-clusterrole
  apiGroup: rbac.authorization.k8s.io
---
# Source: fluent-bit/templates/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: monitor
  labels:
    app: fluent-bit-loki
    chart: fluent-bit-0.0.2
    release: romping-tapir
    heritage: Tiller
  annotations:
    {}

spec:
  selector:
    matchLabels:
      app: fluent-bit-loki
      release: romping-tapir
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: fluent-bit-loki
        release: romping-tapir
      annotations:
        checksum/config: ced25eb2ee16f065206a9a9df947a79825c89e395daf110026333582a761f3bb
        prometheus.io/path: /api/v1/metrics/prometheus
        prometheus.io/port: "2020"
        prometheus.io/scrape: "true"

    spec:
      serviceAccountName: fluent-bit
      containers:
        - name: fluent-bit-loki
          image: "xxxxx/library/grafana/fluent-bit-plugin-loki:latest"   #使用镜像
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: config
              mountPath: /fluent-bit/etc
            - name: run
              mountPath: /run/fluent-bit
            - mountPath: /var/log
              name: varlog
            - mountPath: /var/lib/docker/containers
              name: varlibdockercontainers
              readOnly: true

          ports:
            - containerPort: 2020
              name: http-metrics
          resources:
            {}

      nodeSelector:
        {}

      affinity:
        {}

      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master

      terminationGracePeriodSeconds: 10
      volumes:
        - name: config
          configMap:
            name: fluent-bit
        - name: run
          hostPath:
            path: /run/fluent-bit
        - hostPath:
            path: /var/log
          name: varlog
        - hostPath:
            path: /var/lib/docker/containers
          name: varlibdockercontainers