一、背景
- 当项目中用到集群环境时，一个springboot的应用，会发布到多个tomcat中，在排除故障的时候，必须要每个tomcat都登录上去查看，非常麻烦。所以就有了用filebeat+redis+ELK将分布式日志集中起来，分析的想法。
二、环境介绍
- 系统： Centos7
- 框架：springboot+logback
- 在springboot项目中做到了分不同环境执行不同log配置。dev开发环境只是简单的输出日志在控制台，test以及production环境输出到文件中，每个项目的日志都放在自己相应的文件夹下，并且做到按天分隔日志，当前正在写日志的log文件命令为info.log，使用filebeat去抓取此文件日志。
-
结构
image.png - log-test.xml配置
${projectName}
${CONSOLE_PATTERN}
${logPath}/info.log
${logPath}/info.%d{yyyy-MM-dd}.log
${maxHistory}
true
${CONSOLE_PATTERN}
info
0
${queueSize}
-
贴一下application.properties中配置的projectName
image.png -
最终的日志文件路径如下
image.png - 日志内容如下
2019-04-12 15:20:31.828 [o-9099-exec-702] INFO o.s.web.servlet.DispatcherServlet - FrameworkServlet 'dispatcherServlet': initialization started
2019-04-12 15:20:31.853 [o-9099-exec-702] INFO o.s.web.servlet.DispatcherServlet - FrameworkServlet 'dispatcherServlet': initialization completed in 24 ms
2019-04-12 15:20:31.902 [o-9099-exec-702] ERROR o.s.b.w.servlet.support.ErrorPageFilter - Forwarding to error page from request [/user/test1] due to exception [null]
java.lang.NullPointerException: null
at com.my.test.controller.UserController.test(UserController.java:61)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:209)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:102)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:877)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:783)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:991)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:925)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:974)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:866)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:851)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:109)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:93)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:130)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.access$000(ErrorPageFilter.java:66)
at org.springframework.boot.web.servlet.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.boot.web.servlet.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:80)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:624)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:341)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:783)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:798)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1441)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
三、配置filebeat收集info文件日志
- 编辑filebeat.yml
vi /etc/filebeat/filebeat.yml
- 输入以下内容
###################### Filebeat Configuration Example #########################
# This file is an example configuration file highlighting only the most common
# options. The filebeat.reference.yml file from the same directory contains all the
# supported options with more comments. You can use it as a reference.
#
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html
# For more available modules and options, please see the filebeat.reference.yml sample
# configuration file.
#=========================== Filebeat inputs =============================
filebeat.inputs:
#=========== nginx access-json日志 ==============
- type: log
enabled: true
paths:
- /var/log/nginx/access-json.log #指明读取文件的位置
tags: ["nginx-access"] #用于logstash过滤
#=========== nginx error日志 ==============
- type: log
enabled: true
paths:
- /var/log/nginx/error.log #指明读取文件的位置
tags: ["nginx-error"] #用于logstash过滤
#=========== tomcat access日志 ==============
- type: log
enabled: true
paths:
- /root/server/apache-tomcat-aic/logs/localhost_access_log.Y-M-D.txt #指明tomcat-access日志文件的位置
tags: ["tomcat-access"] #用于logstash过滤
#=========== tomcat job项目日志 ==============
- type: log
enabled: true
paths:
- /root/server/apache-tomcat-aic/logs/aic-job/info.log #指明Springboot项目日志文件的位置
multiline:
pattern: '^\s*(\d{4}|\d{2})\-(\d{2}|[a-zA-Z]{3})\-(\d{2}|\d{4})' # 指定匹配的表达式（匹配以 2017-11-15 08:04:23:889 时间格式开头的字符串）
negate: true # 是否匹配到
match: after # 合并到上一行的末尾, 为了error日志
max_lines: 1000 # 最大的行数
timeout: 30s # 如果在规定的时候没有新的日志事件就不等待后面的日志
tags: ["aic-job"] #用于logstash过滤
#============================= Filebeat modules ===============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# Period on which files under path should be checked for changes
#reload.period: 10s
#==================== Elasticsearch template setting ==========================
setup.template.settings:
index.number_of_shards: 3
#index.codec: best_compression
#_source.enabled: false
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Redis output ------------------------------
output.redis:
hosts: ["192.168.1.110:6379"] #输出到redis的机器
password: "123456"
key: "filebeat:test16" #redis中日志数据的key值ֵ
db: 0
timeout: 5
#================================ Processors =====================================
# Configure processors to enhance or manipulate events generated by the beat.
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
四、配置logstash过滤项目info文件日志
- 新建一个logstash的配置文件
vi /etc/logstash/conf.d/myJob.conf
- 输入以下内容
input {
redis {
data_type =>"list"
key =>"filebeat:test16"
host =>"192.168.1.110"
port => 6379
password => "123456"
threads => "8"
db => 0
#codec => json
}
}
filter {
if "aic-job" in [tags]{
grok {
match => ["message", "%{TIMESTAMP_ISO8601:time}\s* \s*%{NOTSPACE:thread-id}\s* \s*%{LOGLEVEL:level}\s* \s*%{JAVACLASS:class}\s* \- \s*%{JAVALOGMESSAGE:logmessage}\s*"]
}
}
mutate {
remove_field => "log"
remove_field => "beat"
remove_field => "meta"
remove_field => "prospector"
remove_field => "[host][os]"
}
}
output {
if "aic-job" in [tags]{
elasticsearch {
hosts => ["192.168.1.110:9200"]
index => "logstash-test16-tomcat-aic-job-%{+yyyy.MM.dd}"
}
}
}
- 重启logstash，查看启动日志是否报错。
systemctl restart logstash #重启
tail -f /var/log/logstash/logstash-plain.log #查看运行日志
五、配置Kibana展示应用日志
image.png
image.png