Docker安装部署logstash导mysql数据入ES

安装包下载：

1. 官网下载路径：https://artifacts.elastic.co/downloads/logstash/logstash-7.1.0.tar.gz
2. 网盘下载链接：https://pan.baidu.com/s/1-bLS6YUOk93BvtPhrPiUDg 提取码：0poq

构建logstash镜像

1. 解压压缩包：tar -xf logstash-7.1.0.tar.gz
2. 编写镜像配置文件：vi Dockerfile

# 本地构建
FROM java:8
MAINTAINER [email protected]
ADD logstash-7.1.0 /usr/share/logstash
RUN cd /usr/share/logstash
ADD run.sh /run.sh
RUN chmod 755 /*.sh
EXPOSE 5047
CMD ["/run.sh"]

3. 编写logstash启动脚本：vi run.sh

#!/bin/bash
/usr/share/logstash/bin/logstash

4. 编写构建镜像脚本：vi build.sh

docker stop logstash
docker rm logstash
docker rmi custom/logstash-7.1.0
docker build -t custom/logstash-7.1.0 .

5. 启动脚本，构建镜像：sh build.sh
   

使用docker-compose启动logstash

1. 创建文件目录：mkdir -p config lib pipeline
2. 把mysql-connector-java-8.0.21.jar放到lib目录
3. 编写logstash启动配置文件：vi config/logstash.yml

config:
  reload:
    automatic: true
    interval: 3s
xpack:
  management.enabled: false
  monitoring.enabled: false

4. 编写管道配置文件：vi config/pipelines.yml

#- pipeline.id: logstash_dev
#  path.config: "/usr/share/logstash/pipeline/logstash_dev.conf"
- pipeline.id: logstash_mysql
  path.config: "/usr/share/logstash/pipeline/logstash_mysql.conf"

5. 编写mysql管道文件：vi pipeline/logstash_mysql.conf

input {
  jdbc {
      # 数据库连接配置
      jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/member?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC"
      jdbc_user => "root"
      jdbc_password => "123456"

      # 数据库驱动配置
      jdbc_driver_library => "/usr/share/logstash/lib/extra/mysql-connector-java-8.0.21.jar"
      jdbc_driver_class => "com.mysql.jdbc.Driver"
      jdbc_validate_connection => "true"

      jdbc_paging_enabled => "true"
      jdbc_page_size => "1000"
      jdbc_default_timezone => "Asia/Shanghai"

      # 执行的查询语句（全量更新）
      statement => "select * from user_member"

      schedule => "* * * * *"
      use_column_value => true
      tracking_column_type => "numeric"
      tracking_column => "id"

      clean_run => false

      type => "jdbc"
    }

}



output {
  elasticsearch {
        hosts => ["http://127.0.0.1:9200"]
        index => "user_member"
        document_type => "_doc"
        document_id => "%{id}"
    }
    stdout {
        codec => json_lines
    }
}

6. 编写默认管道配置（tcp方式）：vi pipeline/logstash_dev.conf

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 5047
    codec => json_lines
  }
}
filter{
}
output {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "logstash-dev-%{+YYYY.MM.dd}"
    }
    stdout { codec => rubydebug }
}

7. 编写docker-compose配置文件：vi docker-compose.yml

version: "3"
  logstash:
    container_name: logstash
    hostname: logstash
    image: custom/logstash-7.1.0:latest
    privileged: true
    ports:
      - 5047:5047
      - 9600:9600
    volumes:
      - ./logstash/lib/:/usr/share/logstash/lib/extra
      - ./logstash/config/:/usr/share/logstash/config/
      - ./logstash/pipeline/:/usr/share/logstash/pipeline/

8. 启动logstash日志采集服务
   • 启动之前需要将确保elasticsearch服务启动，可参考：Docker部署elasticsearch
   • 使用docker-compose启动：docker-compose up -d
   • 查看启动日志：docker exec -it logstash /bin/bash
9. 在kibana查看数据是否添加到es中