Elk搭建配置
elasticsearch + filebeat + kibana（v7.9.0）
一、elasticsearch安装（略）
cd /opt
# 下载地址
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz
tar -zxvf elasticsearch-7.9.0-linux-x86_64.tar.gz
mv elasticsearch-7.9.0-linux-x86_64 /usr/local
# 配置
vim elasticsearch.yml
# 配置文件参数
# 配置参照【①】
=======================
# 设置vm大小
vim jvm.options
-Xms 1g
-Xmx 1g
###################################################################
# ik 分词插件下载地址
wget https://github.com/medcl/elasticsearch-analysis-ik/archive/v7.9.0.tar.gz
# 安装方式 将 ik包 解压到 es plugins 目录下
【①】elasticsearch.yml
cluster.name: test-app-es
node.name: test-es-node-1
path.data: /Users/liuty/opt/7.9/elasticsearch-7.9.0/data
path.logs: /Users/liuty/opt/7.9/elasticsearch-7.9.0/logs
network.host: 0.0.0.0
transport.tcp.port: 19300
transport.host: 0.0.0.0
http.port: 19200
cluster.initial_master_nodes: ["test-es-node-1"]
# 跨域设置
http.cors.enabled: true
http.cors.allow-origin: "*"
# 禁用xpack验证，否则在连接es时需要使用xpack验证的方式。
xpack.security.enabled: false
二、kibana 安装
cd /opt
# 下载地址
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.9.0-linux-x86_64.tar.gz
tar -zxvf kibana-7.9.0-linux-x86_64.tar.gz
mv kibana-7.9.0-linux-x86_64 kibana-7.9.0
cd kibana-7.9.0
# 配置kibana配置文件
vim config/kibana.yml
# 配置参照【②】
=======================
# 启动 kibana
/opt/kibana-7.9.0/bin/kibana &
# 关闭kibana
ps -ef |grep kibana
kill -pid
【②】kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:19200"]
# es 用户认证
#elasticsearch.username: xxx
#elasticsearch.password: xxx
# Specifies locale to be used for all localizable strings, dates and number formats.
# Supported languages are the following: English - en , by default , Chinese - zh-CN .
# 设置汉化
i18n.locale: "zh-CN"
# 设置kibana 日志路径
logging.dest: /opt/kibana-7.9.0/logs/kibana.log
# 默认值: false 该值设为 true 时，禁止所有日志输出。
# logging.silent:
# 默认值: false 该值设为 true 时，禁止除错误信息除外的所有日志输出。
# logging.quiet:
# 默认值: false 该值设为 true 时，记下所有事件包括系统使用信息和所有请求的日志。
# logging.verbose
三、filebeat安装
# 下载地址
cd /opt
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.9.0-linux-x86_64.tar.gz
tar -zxvf filebeat-7.9.0-linux-x86_64.tar.gz
mv filebeat-7.9.0-linux-x86_64 filebeat-7.9.0
cd filebeat-7.9.0
# 配置自定义配置（以manager-server项目为例）
vim filebeat.yml
# 配置参照【③】
=======================
# 配置自定义日志模版
vim my-fields.yml
# 配置参照【④】
=======================
# 启动 filebeat
nohup ./filebeat -e -c filebeat.yml -d "*" &
# 停止 filebeat
ps -ef |grep filebeat
kill -9 pid
【③】filebeat.yml(以manager-server项目为例)
filebeat.inputs:
##############################【查询-日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.select.log
fields:
type: "select"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【更新-日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.update.log
fields:
type: "update"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【登录-日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.login.log
fields:
type: "login"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【快递管家-回调日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.callback.kdgj.log
fields:
type: "callback.kdgj"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【微信-回调日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.callback.wechat.log
fields:
type: "callback.wechat"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【err-日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.error.log
fields:
type: "error"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
##############################【info-日志】#############################################
- type: log
enabled: true
paths:
- /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.info.log
fields:
type: "info"
server-host: "127.0.0.1"
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
multiline.negate: false
multiline.match: after
multiline.max_lines: 500
### 删除无用字段
processors:
- drop_fields:
fields: ["agent.hostname","agent.id","agent.name","agent.type","agent.version","ecs.version","input.type","log.flags","log.offset"]
#filebeat.config.modules:
# path: ${path.config}/modules.d/*.yml
# reload.enabled: true
setup.template.settings:
index.number_of_shards: 1
#setup.kibana:
# 设置 ilm false，否则自定义索引名称不生效
setup.ilm.enabled: false
output.elasticsearch:
# es 连接配置
hosts: ["localhost:19200"]
indices:
# 【查询-日志】
- index: "manager-select_%{+yyyy-MM-dd}"
when.equals:
fields.type: "select"
# 【更新-日志】
- index: "manager-update_%{+yyyy-MM-dd}"
when.equals:
fields.type: "update"
# 【登录-日志】
- index: "manager-login_%{+yyyy-MM-dd}"
when.equals:
fields.type: "login"
# 【快递管家-回调日志】
- index: "manager-callback.kdgj_%{+yyyy-MM-dd}"
when.equals:
fields.type: "callback.kdgj"
# 【微信-回调日志】
- index: "manager-callback.wechat_%{+yyyy-MM-dd}"
when.equals:
fields.type: "callback.wechat"
# 【err-日志】
- index: "manager-err_%{+yyyy-MM-dd}"
when.equals:
fields.type: "err"
# 【info-日志】
- index: "manager-info_%{+yyyy-MM-dd}"
when.equals:
fields.type: "info"
# 设置模版(mapping||template.json)名称
setup.template.name: "manager"
setup.template.pattern: "manager_*"
#setup.template.overwrite: true
#setup.template.enabled: true
# 设置自定义加载字段
setup.template.fields: "/Users/liuty/opt/7.9/filebeat-7.9.0/my-fields.yml"
logging.level: info
logging.to_files: true
logging.files:
path: /Users/liuty/opt/7.9/filebeat-7.9.0
name: filebeat.log
# 设置文件最多生成数量
keepfiles: 7
# 设置文件权限，默认0600
permissions: 0644
# 设置文件最大大小
rotateeverybytes: 104857600
【④】my-fields.yml(自定义字段配置)
# WARNING! Do not edit this file directly, it was generated by the ECS project,
# based on ECS version 1.5.0.
# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields.
- key: ecs
title: ECS
description: ECS Fields.
fields:
- name: '@timestamp'
level: core
required: true
type: date
description: 'Date/time when the event originated.
This is the date/time extracted from the event, typically representing when
the event was generated by the source.
If the event source has no original timestamp, this value is typically populated
by the first time the event was received by the pipeline.
Required field for all events.'
example: '2016-05-23T08:05:34.853Z'
- name: message
level: core
type: text
description: 'For log events the message field contains the log message, optimized
for viewing in a log viewer.
For structured logs without an original message field, other fields can be concatenated
to form a human-readable summary of the event.
If multiple messages exist, they can be combined into one message.'
example: Hello World
- name: agent
title: Agent
group: 2
description: 'The agent fields contain the data about the software entity, if
any, that collects, detects, or observes events on a host, or takes measurements
on a host.
Examples include Beats. Agents may also run on observers. ECS agent.* fields
shall be populated with details of the agent running on the host or observer
where the event happened or the measurement was taken.'
footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat.
For APM, it is the agent running in the app/service. The agent information does
not change if data is sent through queuing systems like Kafka, Redis, or processing
systems such as Logstash or APM Server.'
type: group
fields:
- name: name
level: core
type: keyword
ignore_above: 1024
description: 'Custom name of the agent.
This is a name that can be given to an agent. This can be helpful if for example
two Filebeat instances are running on the same host but a human readable separation
is needed on which Filebeat instance data is coming from.
If no name is given, the name is often left empty.'
example: foo
- key: log
title: Log file content
description: >
Contains log file lines.
fields:
- name: log.file.path
type: keyword
required: false
description: >
The file from which the line was read. This field contains the absolute path to the file.
For example: `/var/log/system.log`.
- name: log.offset
type: long
required: false
description: >
The file offset the reported line starts at.
四、kibana操作及查询
1.查询日志中 路由为loginNew 并且 请求时间是 2020-08-28 09:11:10 的日志(kql)
message :"loginNew" and message :"Date : 2020-08-28 09:11:10"
result:
2020-08-28 09:11:10,179 [http-nio-8081-exec-9] INFO loginLogger 283 -
======Start======
Date : 2020-08-28 09:11:10
OperatorId :
URL : http://127.0.0.1:8081/manager-server/sysAdmin/cms/loginNew
HTTP Method : POST
Class Method : com.memory.cms.controller.SysAdminCmsController.loginNew
IP : 127.0.0.1
Request Args : {"project":"内容模型管理系统","pwd":"123456","loginName":"memory"}
+++++++++++++++++++++++++
Response Args : {"state":"success","recode":0,"msg":"success","data":{"sysAdminObj":{"birthday":"","address":"","loginname":"memory","nickName":"管理员","sex":"男","name":"系统管理员","logo":"stock/sysAdmin/1/logo_20200728_142732.png","tel":"1","id":"1","email":""},"roleLimitsList":["ftQRFseQ","CPv7xlsp","iRtoXNgb","sh07yD7c","IeORS7rQ","uJSDARfn","RIfFB0VV","hwVlDezR","WuLcDCJE","mGADJLZC","syqnlMu4","vmyRDZSN","zJ8DviNs","9N3w2baX","vgq3wYpn","7cevLe3o","cbO0rWpp","ZSPc1JLN","sW9ldL3h","YLj75P7v","hVmy5lDH","bIdm3N07","SQPyXCKO","r68bGlME","lcWLNUAD","jIYdQ9Sb","BIcqlj3E","qSHXQ4xk","PWsFBRF9","X5AEDFU8","IEbak87F","5p6RThvG","QM8SYzgB","wkCAAXiu","nrnQvLME","pMsEZLEl","NKDQmYQZ"],"roleDicList":[{"id":"hVmy5lDH","parent":"系统管理","path":"system","meta":"角色管理","childrenSort":100,"parentSort":1,"page":"role","hasPatent":1,"project":"内容模型管理系统","hasChildren":0,"parentName":"","parentId":""}],"fileUrl":"http://192.168.1.200:8091/file/","accessToken":"eyJhbGciOiJSUzI1NiJ9.eyJhdWQiOiI1OGNjZmM4ZDI5MDM0MTg5YjUxOThlYmMzZTg2ZGI2ZCIsInRpbWUiOjE1OTg1NzcwNzAwNDQsImFkbWluVHlwZSI6ImNtcyIsInN5c0FkbWluSWQiOiIxIn0.1kbEmBqjScO3u61jxy7tQUlha5hr4GZGhn9R06N7PxL1vuvvWCxYVdCUJ2W7tqhH_Fvu3RO4XI5s4iq6oaKpCHiQ3IuLu0diMlKvv-CiAmmbE3kgfkqvoVu7Yiq-0GiD0mEVxJr7x91XPI7Ue6c4k1USlfYDoYkx6zg8wAOeY6k"}}
Times : 237ms
======End======
2.查询返回error的接口请求(kql)
message:"Response Args" and state and error
result:
2020-08-28 09:10:45,523 [http-nio-8081-exec-7] INFO loginLogger 283 -
======Start======
Date : 2020-08-28 09:10:45
OperatorId :
URL : http://127.0.0.1:8081/manager-server/sysAdmin/cms/loginNew
HTTP Method : POST
Class Method : com.memory.cms.controller.SysAdminCmsController.loginNew
IP : 127.0.0.1
Request Args : {"project":"内容模型管理系统","pwd":"123456","loginName":"ganxiqing"}
+++++++++++++++++++++++++
Response Args : {"state":"error","recode":-1,"msg":"用户名不存在或密码错误！"}
Times : 31ms
======End======
3.模糊匹配，查询所有 logn（login logan logen...） 的日志(Lucene*)
log?n/log*n
4.java 异常日志(kql)
message :"Caused by"
result:
2020-08-26 17:01:55,771 [main] ERROR o.s.boot.SpringApplication 858 - Application run failed
java.lang.NoClassDefFoundError: org/elasticsearch/transport/client/PreBuiltTransportClient
at com.memory.appes.ElasticSearchUtil.getClient(ElasticSearchUtil.java:111)
at com.memory.appes.tcp.IndexElasticSearch.existsIndex(IndexElasticSearch.java:25)
at com.memory.appes.ElasticSearchUtil.initIndexs(ElasticSearchUtil.java:94)
at com.memory.ApplicationListener.callback(ApplicationListener.java:54)
at com.memory.appcommon.listener.Listener.onApplicationEvent(Listener.java:26)
at com.memory.appcommon.listener.Listener.onApplicationEvent(Listener.java:17)
at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:398)
at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:355)
at org.springframework.boot.context.event.EventPublishingRunListener.started(EventPublishingRunListener.java:99)
at org.springframework.boot.SpringApplicationRunListeners.started(SpringApplicationRunListeners.java:72)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:323)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1260)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1248)
at com.memory.AppControllerManagerApplication.main(AppControllerManagerApplication.java:19)
Caused by: java.lang.ClassNotFoundException: org.elasticsearch.transport.client.PreBuiltTransportClient
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 17 common frames omitted