Centos下Elk搭建配置

Elk搭建配置

elasticsearch + filebeat + kibana（v7.9.0）

一、elasticsearch安装（略）

cd /opt
# 下载地址
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz
tar -zxvf elasticsearch-7.9.0-linux-x86_64.tar.gz
mv elasticsearch-7.9.0-linux-x86_64  /usr/local
# 配置
vim elasticsearch.yml
# 配置文件参数
# 配置参照【①】
=======================

# 设置vm大小
vim jvm.options
    -Xms 1g
    -Xmx 1g
    
###################################################################
# ik 分词插件下载地址
wget https://github.com/medcl/elasticsearch-analysis-ik/archive/v7.9.0.tar.gz
# 安装方式 将 ik包 解压到 es plugins 目录下

【①】elasticsearch.yml

cluster.name: test-app-es
node.name: test-es-node-1
path.data: /Users/liuty/opt/7.9/elasticsearch-7.9.0/data
path.logs: /Users/liuty/opt/7.9/elasticsearch-7.9.0/logs
network.host: 0.0.0.0
transport.tcp.port: 19300
transport.host: 0.0.0.0
http.port: 19200

cluster.initial_master_nodes: ["test-es-node-1"]
# 跨域设置
http.cors.enabled:    true
http.cors.allow-origin:  "*"
# 禁用xpack验证，否则在连接es时需要使用xpack验证的方式。
xpack.security.enabled: false

二、kibana 安装

cd /opt
# 下载地址
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.9.0-linux-x86_64.tar.gz
tar -zxvf kibana-7.9.0-linux-x86_64.tar.gz
mv kibana-7.9.0-linux-x86_64 kibana-7.9.0
cd kibana-7.9.0
# 配置kibana配置文件
vim config/kibana.yml

# 配置参照【②】
=======================
 
 
 # 启动 kibana
 /opt/kibana-7.9.0/bin/kibana &
 # 关闭kibana
 ps -ef |grep kibana
 kill -pid
 

【②】kibana.yml

 server.port: 5601
 server.host: "0.0.0.0"
 elasticsearch.hosts: ["http://localhost:19200"]
 # es 用户认证
 #elasticsearch.username: xxx
 #elasticsearch.password: xxx

 # Specifies locale to be used for all localizable strings, dates and number formats.
 # Supported languages are the following: English - en , by default , Chinese - zh-CN .
 # 设置汉化
 i18n.locale: "zh-CN"
 # 设置kibana 日志路径
 logging.dest: /opt/kibana-7.9.0/logs/kibana.log
 # 默认值: false 该值设为 true 时，禁止所有日志输出。
 # logging.silent:
 # 默认值: false 该值设为 true 时，禁止除错误信息除外的所有日志输出。
 # logging.quiet:
 # 默认值: false 该值设为 true 时，记下所有事件包括系统使用信息和所有请求的日志。
 # logging.verbose

三、filebeat安装

# 下载地址
cd /opt
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.9.0-linux-x86_64.tar.gz
tar -zxvf filebeat-7.9.0-linux-x86_64.tar.gz
mv filebeat-7.9.0-linux-x86_64 filebeat-7.9.0
cd filebeat-7.9.0

# 配置自定义配置（以manager-server项目为例）
vim filebeat.yml
# 配置参照【③】
=======================
# 配置自定义日志模版
vim my-fields.yml
# 配置参照【④】
=======================
# 启动 filebeat 
nohup ./filebeat -e -c filebeat.yml -d "*" &
# 停止 filebeat
ps -ef |grep filebeat
kill -9 pid
 

【③】filebeat.yml(以manager-server项目为例)

filebeat.inputs:

##############################【查询-日志】#############################################
- type: log
  enabled: true
  paths:
   - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.select.log
  fields:
    type: "select"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【更新-日志】#############################################
- type: log
  enabled: true
  paths:
   - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.update.log
  fields:
    type: "update"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【登录-日志】#############################################
- type: log
  enabled: true
  paths:
   - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.login.log
  fields:
    type: "login"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【快递管家-回调日志】#############################################
- type: log
  enabled: true
  paths:
  - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.callback.kdgj.log
  fields:
    type: "callback.kdgj"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【微信-回调日志】#############################################
- type: log
  enabled: true
  paths:
  - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.callback.wechat.log
  fields:
    type: "callback.wechat"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【err-日志】#############################################
- type: log
  enabled: true
  paths:
  - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.error.log
  fields:
    type: "error"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500
##############################【info-日志】#############################################
- type: log
  enabled: true
  paths:
  - /Users/liuty/gwzz/svn/manager-server/app-controller-manager_2020-08-15_week_lty/logs-manager-server/*.info.log
  fields:
    type: "info"
    server-host: "127.0.0.1"
  multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^\D'
  multiline.negate: false
  multiline.match: after
  multiline.max_lines: 500

### 删除无用字段
processors:
- drop_fields:
   fields: ["agent.hostname","agent.id","agent.name","agent.type","agent.version","ecs.version","input.type","log.flags","log.offset"]
#filebeat.config.modules:
#  path: ${path.config}/modules.d/*.yml
#  reload.enabled: true


setup.template.settings:
  index.number_of_shards: 1

#setup.kibana:

# 设置 ilm false，否则自定义索引名称不生效
setup.ilm.enabled: false
output.elasticsearch:
  # es 连接配置
  hosts: ["localhost:19200"]
  indices:
  
    # 【查询-日志】
    - index: "manager-select_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "select"
          
    # 【更新-日志】
    - index: "manager-update_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "update"
        
    # 【登录-日志】
    - index: "manager-login_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "login"
        
    # 【快递管家-回调日志】
    - index: "manager-callback.kdgj_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "callback.kdgj"
        
    # 【微信-回调日志】
    - index: "manager-callback.wechat_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "callback.wechat"
        
    # 【err-日志】
    - index: "manager-err_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "err"
        
    # 【info-日志】
    - index: "manager-info_%{+yyyy-MM-dd}"
      when.equals:
        fields.type: "info"
        
  
# 设置模版(mapping||template.json)名称
setup.template.name: "manager"
setup.template.pattern: "manager_*"
#setup.template.overwrite: true
#setup.template.enabled: true
# 设置自定义加载字段
setup.template.fields: "/Users/liuty/opt/7.9/filebeat-7.9.0/my-fields.yml"

logging.level: info
logging.to_files: true
logging.files:
  path: /Users/liuty/opt/7.9/filebeat-7.9.0
  name: filebeat.log
  # 设置文件最多生成数量
  keepfiles: 7
  # 设置文件权限，默认0600
  permissions: 0644
  # 设置文件最大大小
  rotateeverybytes: 104857600

【④】my-fields.yml(自定义字段配置)

# WARNING! Do not edit this file directly, it was generated by the ECS project,
# based on ECS version 1.5.0.
# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields.

- key: ecs
  title: ECS
  description: ECS Fields.
  fields:
  - name: '@timestamp'
    level: core
    required: true
    type: date
    description: 'Date/time when the event originated.

      This is the date/time extracted from the event, typically representing when
      the event was generated by the source.

      If the event source has no original timestamp, this value is typically populated
      by the first time the event was received by the pipeline.

      Required field for all events.'
    example: '2016-05-23T08:05:34.853Z'
  - name: message
    level: core
    type: text
    description: 'For log events the message field contains the log message, optimized
      for viewing in a log viewer.

      For structured logs without an original message field, other fields can be concatenated
      to form a human-readable summary of the event.

      If multiple messages exist, they can be combined into one message.'
    example: Hello World

  - name: agent
    title: Agent
    group: 2
    description: 'The agent fields contain the data about the software entity, if
      any, that collects, detects, or observes events on a host, or takes measurements
      on a host.

      Examples include Beats. Agents may also run on observers. ECS agent.* fields
      shall be populated with details of the agent running on the host or observer
      where the event happened or the measurement was taken.'
    footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat.
      For APM, it is the agent running in the app/service. The agent information does
      not change if data is sent through queuing systems like Kafka, Redis, or processing
      systems such as Logstash or APM Server.'
    type: group
    fields:
    - name: name
      level: core
      type: keyword
      ignore_above: 1024
      description: 'Custom name of the agent.

        This is a name that can be given to an agent. This can be helpful if for example
        two Filebeat instances are running on the same host but a human readable separation
        is needed on which Filebeat instance data is coming from.

        If no name is given, the name is often left empty.'
      example: foo
    
- key: log
  title: Log file content
  description: >
    Contains log file lines.
  fields:

    - name: log.file.path
      type: keyword
      required: false
      description: >
        The file from which the line was read. This field contains the absolute path to the file.
        For example: `/var/log/system.log`.
        
    - name: log.offset
      type: long
      required: false
      description: >
        The file offset the reported line starts at.

四、kibana操作及查询

1.查询日志中 路由为loginNew 并且 请求时间是 2020-08-28 09:11:10 的日志(kql)

message :"loginNew" and message :"Date : 2020-08-28 09:11:10"

result:

    2020-08-28 09:11:10,179 [http-nio-8081-exec-9] INFO  loginLogger 283 - 
 ======Start====== 
 Date : 2020-08-28 09:11:10
 OperatorId : 
 URL : http://127.0.0.1:8081/manager-server/sysAdmin/cms/loginNew  
 HTTP Method : POST  
 Class Method : com.memory.cms.controller.SysAdminCmsController.loginNew 
 IP : 127.0.0.1 
 Request Args : {"project":"内容模型管理系统","pwd":"123456","loginName":"memory"} 
 +++++++++++++++++++++++++  
 Response Args : {"state":"success","recode":0,"msg":"success","data":{"sysAdminObj":{"birthday":"","address":"","loginname":"memory","nickName":"管理员","sex":"男","name":"系统管理员","logo":"stock/sysAdmin/1/logo_20200728_142732.png","tel":"1","id":"1","email":""},"roleLimitsList":["ftQRFseQ","CPv7xlsp","iRtoXNgb","sh07yD7c","IeORS7rQ","uJSDARfn","RIfFB0VV","hwVlDezR","WuLcDCJE","mGADJLZC","syqnlMu4","vmyRDZSN","zJ8DviNs","9N3w2baX","vgq3wYpn","7cevLe3o","cbO0rWpp","ZSPc1JLN","sW9ldL3h","YLj75P7v","hVmy5lDH","bIdm3N07","SQPyXCKO","r68bGlME","lcWLNUAD","jIYdQ9Sb","BIcqlj3E","qSHXQ4xk","PWsFBRF9","X5AEDFU8","IEbak87F","5p6RThvG","QM8SYzgB","wkCAAXiu","nrnQvLME","pMsEZLEl","NKDQmYQZ"],"roleDicList":[{"id":"hVmy5lDH","parent":"系统管理","path":"system","meta":"角色管理","childrenSort":100,"parentSort":1,"page":"role","hasPatent":1,"project":"内容模型管理系统","hasChildren":0,"parentName":"","parentId":""}],"fileUrl":"http://192.168.1.200:8091/file/","accessToken":"eyJhbGciOiJSUzI1NiJ9.eyJhdWQiOiI1OGNjZmM4ZDI5MDM0MTg5YjUxOThlYmMzZTg2ZGI2ZCIsInRpbWUiOjE1OTg1NzcwNzAwNDQsImFkbWluVHlwZSI6ImNtcyIsInN5c0FkbWluSWQiOiIxIn0.1kbEmBqjScO3u61jxy7tQUlha5hr4GZGhn9R06N7PxL1vuvvWCxYVdCUJ2W7tqhH_Fvu3RO4XI5s4iq6oaKpCHiQ3IuLu0diMlKvv-CiAmmbE3kgfkqvoVu7Yiq-0GiD0mEVxJr7x91XPI7Ue6c4k1USlfYDoYkx6zg8wAOeY6k"}} 
 Times : 237ms 
 ======End====== 

2.查询返回error的接口请求(kql)

message:"Response Args" and state and error

result:

2020-08-28 09:10:45,523 [http-nio-8081-exec-7] INFO  loginLogger 283 - 
 ======Start====== 
 Date : 2020-08-28 09:10:45
 OperatorId : 
 URL : http://127.0.0.1:8081/manager-server/sysAdmin/cms/loginNew  
 HTTP Method : POST  
 Class Method : com.memory.cms.controller.SysAdminCmsController.loginNew 
 IP : 127.0.0.1 
 Request Args : {"project":"内容模型管理系统","pwd":"123456","loginName":"ganxiqing"} 
 +++++++++++++++++++++++++  
 Response Args : {"state":"error","recode":-1,"msg":"用户名不存在或密码错误！"} 
 Times : 31ms 
 ======End====== 

3.模糊匹配，查询所有 logn（login logan logen...） 的日志(Lucene*)

log?n/log*n 

4.java 异常日志(kql)

message :"Caused by" 

result:

2020-08-26 17:01:55,771 [main] ERROR o.s.boot.SpringApplication 858 - Application run failed
java.lang.NoClassDefFoundError: org/elasticsearch/transport/client/PreBuiltTransportClient
    at com.memory.appes.ElasticSearchUtil.getClient(ElasticSearchUtil.java:111)
    at com.memory.appes.tcp.IndexElasticSearch.existsIndex(IndexElasticSearch.java:25)
    at com.memory.appes.ElasticSearchUtil.initIndexs(ElasticSearchUtil.java:94)
    at com.memory.ApplicationListener.callback(ApplicationListener.java:54)
    at com.memory.appcommon.listener.Listener.onApplicationEvent(Listener.java:26)
    at com.memory.appcommon.listener.Listener.onApplicationEvent(Listener.java:17)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
    at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
    at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:398)
    at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:355)
    at org.springframework.boot.context.event.EventPublishingRunListener.started(EventPublishingRunListener.java:99)
    at org.springframework.boot.SpringApplicationRunListeners.started(SpringApplicationRunListeners.java:72)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:323)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1260)
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1248)
    at com.memory.AppControllerManagerApplication.main(AppControllerManagerApplication.java:19)
Caused by: java.lang.ClassNotFoundException: org.elasticsearch.transport.client.PreBuiltTransportClient
    at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 17 common frames omitted