实现fluentd采集docker的日志输出到minio

需求

1. 实现采集日志到mino
2. 以容器的id或者name为文件夹进行分别存储

参考的网站

https://my.oschina.net/xiaominmin/blog/1796742

https://docs.fluentd.org/buffer/file#parameters

用到的插件

编写fluent.conf，使其支持不同容器文件写到不同的文件夹

https://github.com/tagomoris/fluent-plugin-forest

编译镜像

https://github.com/fluent/fluentd-docker-image

支持写文件到minio

https://github.com/fluent/fluent-plugin-s3

镜像build

Dockerfile内容

# AUTOMATICALLY GENERATED
# DO NOT EDIT THIS FILE DIRECTLY, USE /Dockerfile.template.erb

FROM alpine:3.9
LABEL maintainer "Fluentd developers "
LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.9.3"

COPY repositories /etc/apk
# Do not split this into multiple RUN!
# Docker creates a layer for every RUN-Statement
# therefore an 'apk delete' has no effect
RUN apk update \
 && apk add  \
        ca-certificates \
        ruby ruby-irb ruby-etc ruby-webrick \
        tini \
 && apk add  --virtual .build-deps \
        build-base linux-headers \
        ruby-dev gnupg \
 && echo 'gem: --no-document' >> /etc/gemrc \
 && gem install oj -v 3.8.1 \
 && gem install json -v 2.3.0 \
 && gem install async-http -v 0.50.0 \
 && gem install ext_monitor -v 0.1.2 \
 && gem install fluentd -v 1.9.3 \
 && gem install bigdecimal -v 1.4.4 \
 && gem install fluent-plugin-s3 -v 1.0.0 --no-document \
 && gem install fluent-plugin-forest -v 0.3.3 \
 && apk del .build-deps \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

RUN addgroup -S fluent && adduser -S -g fluent fluent \
    # for log storage (maybe shared with host)
    && mkdir -p /fluentd/log \
    # configuration/plugins path (default: copied from .)
    && mkdir -p /fluentd/etc /fluentd/plugins \
    && chown -R fluent /fluentd && chgrp -R fluent /fluentd
    COPY fluent.conf /fluentd/etc/
COPY entrypoint.sh /bin/


ENV FLUENTD_CONF="fluent.conf"

ENV LD_PRELOAD=""
EXPOSE 24224 5140

USER fluent
ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
CMD ["fluentd"]

entrypoint.sh文件

需要一些环境变量，可在build时加入

!/bin/sh
  
#source vars if file exists
DEFAULT=/etc/default/fluentd
sed -i "s/MACHINE_NAME/${MACHINE_NAME}/g" /fluentd/etc/fluent.conf
sed -i "s/MINIO_KEY_ID/${MINIO_KEY_ID}/g" /fluentd/etc/fluent.conf
sed -i "s/MINIO_SEC_KEY/${MINIO_SEC_KEY}/g" /fluentd/etc/fluent.conf
sed -i "s/MINIO_BUCKET/${MINIO_BUCKET}/g" /fluentd/etc/fluent.conf
sed -i "s/MINIO_HOST/${MINIO_HOST}/g" /fluentd/etc/fluent.conf
sed -i "s/MINIO_PORT/${MINIO_PORT}/g" /fluentd/etc/fluent.conf


if [ -r $DEFAULT ]; then
    set -o allexport
    . $DEFAULT
    set +o allexport
fi

# If the user has supplied only arguments append them to `fluentd` command
if [ "${1#-}" != "$1" ]; then
    set -- fluentd "$@"
fi

# If user does not supply config file or plugins, use the default
if [ "$1" = "fluentd" ]; then
    if ! echo $@ | grep ' \-c' ; then
       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
    fi

    if ! echo $@ | grep ' \-p' ; then
       set -- "$@" -p /fluentd/plugins
    fi
fi

exec "$@"

fluent.config

  @type  forward
  @label @mainstream
  port  24224
  bind 0.0.0.0



  
   @type record_transformer
   
     host_param "#{Socket.gethostname}"
     machine_name "MACHINE_NAME"
   
  
  
     @type forest
     subtype s3
     
  

收集docker日志

docker-compose.yml的写法

version: '3.7'
x-logging:
  &fluent-logging
  driver: fluentd
  options:
    fluentd-address: ${FLUENTD_SERVER_IP}:${FLUENTD_SERVER_PORT}
    tag: "docker.{{.Name}}"
    fluentd-async-connect: 'true'
    mode: non-blocking
    fluentd-sub-second-precision: 'true'
services:
  example:
    container_name: example
    environment:
      MACHINE_NAME: ${name}
      MINIO_BUCKET: ${bucket}
      MINIO_HOST: 127.0.0.1
      MINIO_KEY_ID: test
      MINIO_PORT: 9000
      MINIO_SEC_KEY: Exxxx
    image: fluentd/fluentd:1.0
    logging: *fluent-logging
    ports:
      - ${FLUENTD_SERVER_PORT}:24224
    privileged: true
    restart: alway