SpringBoot集成ELK实现日志收集实践

一、Docker Compose搭建ELK

Elasticsearch默认使用mmapfs目录来存储索引。操作系统默认的mmap计数太低可能导致内存不足，我们可以使用下面这条命令来增加内存：

sysctl -w vm.max_map_count=262144

创建Elasticsearch数据挂载路径：

mkdir -p /home/elk/elasticsearch/data

对该路径授予777权限：

chmod 777 /home/elk/elasticsearch/data

创建Elasticsearch插件挂载路径：

mkdir -p /home/elk/elasticsearch/plugins

创建Logstash配置文件存储路径：

mkdir -p /home/elk/logstash

在该路径下创建logstash-febs.conf配置文件

vim /home/elk/logstash/logstash-febs.conf

内容如下所示：

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4560
    codec => json_lines
  }
}
output {
  elasticsearch {
    hosts => "es:9200"
    index => "boottest-logstash-%{+YYYY.MM.dd}"
  }
}

在/home/elk目录下创建docker-compose.yml文件：

vim /home/elk/docker-compose.yml

内容如下：

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.8.1
    container_name: elasticsearch
    environment:
      - "cluster.name=elasticsearch" #集群名称为elasticsearch
      - "discovery.type=single-node" #单节点启动
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m" #jvm内存分配为512MB
    volumes:
      - /home/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
      - /home/elk/elasticsearch/data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
  kibana:
    image: kibana:7.8.1
    container_name: kibana
    links:
      - elasticsearch:es #配置elasticsearch域名为es
    depends_on:
      - elasticsearch
    environment:
      - "elasticsearch.hosts=http://es:9200" #因为上面配置了域名，所以这里可以简写为http://es:9200
    ports:
      - 5601:5601
  logstash:
    image: logstash:7.8.1
    container_name: logstash
    volumes:
      - /home/elk/logstash/logstash-febs.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch
    links:
      - elasticsearch:es
    ports:
      - 4560:4560

使用如下命令启动：

docker-compose up -d

docker ps 观察容器运行情况

Logstash中安装json_lines插件，使用如下命令进入到Logstash容器中并安装插件：

#进入容器
docker exec -it logstash /bin/bash
#安装插件
logstash-plugin install logstash-codec-josn_lines
#安装成功后exit退出容器
exit

汉化Kibana服务

#进入容器
docker exec -it kibana /bin/bash
#编辑配置文件
vi /opt/kibana/config/kibana.yml
#加上一行配置
i18n.locale: zh-CN

重启kibana容器，使用浏览器访问http://192.168.33.10:5601便可以看到Kibana管理界面：

二、SpringBoot项目创建

使用idea创建一个初始化项目

pom文件内容如下：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0modelVersion>
    <parent>
        <groupId>org.springframework.bootgroupId>
        <artifactId>spring-boot-starter-parentartifactId>
        <version>2.3.2.RELEASEversion>
        <relativePath/> 
    parent>
    <groupId>com.elk.lggroupId>
    <artifactId>boot_elkartifactId>
    <version>0.0.1-SNAPSHOTversion>
    <name>boot_elkname>
    <description>Demo project for Spring Bootdescription>

    <properties>
        <java.version>1.8java.version>
    properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starterartifactId>
        dependency>

        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-webartifactId>
        dependency>

        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-testartifactId>
            <scope>testscope>
            <exclusions>
                <exclusion>
                    <groupId>org.junit.vintagegroupId>
                    <artifactId>junit-vintage-engineartifactId>
                exclusion>
            exclusions>
        dependency>

        
        <dependency>
            <groupId>net.logstash.logbackgroupId>
            <artifactId>logstash-logback-encoderartifactId>
            <version>5.3version>
        dependency>

        <dependency>
            <groupId>org.projectlombokgroupId>
            <artifactId>lombokartifactId>
            <version>1.18.4version>
        dependency>
    dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.bootgroupId>
                <artifactId>spring-boot-maven-pluginartifactId>
            plugin>
        plugins>
    build>

project>

添加logback.xml配置文件，内容如下：

<configuration scan="true" scanPeriod="60 seconds" debug="false">
    <springProperty scope="context" name="springAppName" source="spring.application.name"/>
    <property name="log.path" value="log/boot_elk"/>
    <property name="log.maxHistory" value="15"/>
    <property name="log.colorPattern"
              value="%magenta(%d{yyyy-MM-dd HH:mm:ss}) %highlight(%-5level) %boldCyan(${springAppName:-}) %yellow(%thread) %green(%logger) %msg%n"/>
    <property name="log.pattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5level ${springAppName:-} %thread %logger %msg%n"/>

    
    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${log.colorPattern}pattern>
        encoder>
    appender>

    
    <appender name="file_info" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/info/info.%d{yyyy-MM-dd}.logfileNamePattern>
            <MaxHistory>${log.maxHistory}MaxHistory>
        rollingPolicy>
        <encoder>
            <pattern>${log.pattern}pattern>
        encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>INFOlevel>
            <onMatch>ACCEPTonMatch>
            <onMismatch>DENYonMismatch>
        filter>
    appender>

    <appender name="file_error" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}/error/error.%d{yyyy-MM-dd}.logfileNamePattern>
        rollingPolicy>
        <encoder>
            <pattern>${log.pattern}pattern>
        encoder>
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERRORlevel>
            <onMatch>ACCEPTonMatch>
            <onMismatch>DENYonMismatch>
        filter>
    appender>

    
    <appender name="logstash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>192.168.2.2:4560destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder"/>
    appender>

    <root level="debug">
        <appender-ref ref="console"/>
    root>

    <root level="info">
        <appender-ref ref="file_info"/>
        <appender-ref ref="file_error"/>
        <appender-ref ref="logstash" />
    root>
configuration>

创建一个测试接口

package com.elk.lg.controller;

import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RequestMapping
@RestController
@Slf4j
public class TestController {

    @GetMapping("/testLog")
    public void testLog(){
        log.debug("message0");
        log.info("message1");
        log.warn("message2");
        log.error("message3");
    }

    @GetMapping("/errorLog")
    public void errorLog(){
        int rs = 1/0;
    }

}

调用测试接口输出相关日志：

三、连接Kibana和Elasticsearch

如上图可以查看到收集到的日志

四、示例代码压缩包下载（无需积分）

传送门