ELk-filebest

image.png

input 我们要采集的日志文件路径, 收割机 harvester 监听文件的变化 --> splooer程序 --> 转发 es | logstash | kafka | redis

image.png

filebeat.inputs:  
  - type: stdin         #标准输入    
    enabled: true       #启用
output.console:         #标准输出   
  pretty: true  
  enable: true

image.png

#将文件最新发生变化的内容,存入ES
[root@web01 ~]# cat /etc/filebeat/file.yml 
filebeat.inputs:  
  - type: log    
    paths: /var/log/nginx/access.log    
    enabled: true

output.elasticsearch:  
  hosts: ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]

收集系统日志
特别分散--> syslog --> ﬁle.txt

1.减少无用的数据
2.调整索引名称
3.测试调整模板,设定分片

[root@web01 ~]# cat /etc/filebeat/filebeat.yml
 filebeat.inputs: 
  - type: log  
    enabled: true  
    paths:    
      - /var/log/oldxu.log  
    include_lines: ['^ERR', '^WARN', 'sshd']   #只看指定的 日志

output.elasticsearch:  
  hosts: 
["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]  index: "system-%{[agent.version]}-%{+yyyy.MM.dd}"
setup.ilm.enabled: false 
setup.template.name: system   #索引关联的模板名称 
setup.template.pattern: system-*
方式一:
###设定system模板的分片数和副本数 
#setup.template.settings:            #定义索引分片数和副本 
#  index.number_of_shards: 3 
#  index.number_of_replicas: 1
方式二:
"number_of_routing_shards": "30",   
"number_of_shards": "10",   
"number_of_replicas": "1",
1.修改system模板   --->  添加 shards 分片数数 量,replicas的数量    
2.删除模板关联的索引    
3.删除filebeat自行指定的分片数和副本数    
4.重启filebeat    
5.产生新的日志

image.png

收集Nginx

vim /etc/nginx/nginx.conf
log_format json '{ "time_local": "$time_local", '                         
 '"remote_addr": "$remote_addr", '                          
'"referer": "$http_referer", '                          
'"request": "$request", '                          
'"status": $status, '                         
'"bytes": $body_bytes_sent, '                          
'"agent": "$http_user_agent", '                          
'"x_forwarded": "$http_x_forwarded_for", '                          
'"up_addr": "$upstream_addr",'                          
'"up_host": "$upstream_http_host",'                          
'"upstream_time": "$upstream_response_time",'                         
 '"request_time": "$request_time"'    3
'}';
    access_log /var/log/nginx/access.log json;

配置ﬁlebeat

[root@web01 filebeat]# cat /etc/filebeat/filebeat.yml
filebeat.inputs: 
  - type: log  
    enabled: true  
    paths:    
      - /var/log/nginx/access.log  
    json.keys_under_root: true    #默认Flase，还会将 
    json解析 的日志存储至messages字段  json.overwrite_keys: true     #覆盖默认的key，使用自定义 json格式的key

output.elasticsearch:  
  hosts: ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]  index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
setup.ilm.enabled: false  
setup.template.name: nginx   #索引关联的模板名称 
setup.template.pattern: nginx-*

收集nginx访问日志和错误日志

filebeat.inputs:

- type: log
  enabled: true
  paths:
  - /var/log/nginx/access.log
    json.keys_under_root: true    #默认Flase，还会将json解析的日志存储至messages字段
    json.overwrite_keys: true     #覆盖默认的key，使用自定义json格式的key
    tags: ["access"]

- type: log
  enabled: true
  paths:
  - /var/log/nginx/error.log
    tags: ["error"]



output.elasticsearch:
  hosts: ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
  indices:
    - index: "nginx-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "access"

    - index: "nginx-error-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "error"

setup.ilm.enabled: false
setup.template.name: nginx   #索引关联的模板名称
setup.template.pattern: nginx-*

收集nginx多个虚拟主机的日志
elk.oldxu.com
bk.oldxu.com
bs.oldxu.com
error日志

image.png

1.虚拟主机

[root@web01 conf.d]# cat elk.oldxu.com.conf 
server {    
        listen 80;    
        server_name elk.oldxu.com;    
        root /code/elk;    
        access_log /var/log/nginx/elk.oldxu.com.log json;
    location / {        
            index index.html;   
 } }
[root@web01 conf.d]# cat bs.oldxu.com.conf 
server {    
        listen 80;    
        server_name bs.oldxu.com;
         root /code/bs;    
        access_log /var/log/nginx/bs.oldxu.com.log json;
    location / {        
        index index.html;    
} }
[root@web01 conf.d]# cat bk.oldxu.com.conf 
server {    
        listen 80;    
        server_name bk.oldxu.com;    
        root /code/bk;    
        access_log /var/log/nginx/bk.oldxu.com.log json;
    location / {        
        index index.html;    
}}

2.测试,模拟产生日志

[root@web01 conf.d]# curl -H Host:elk.oldxu.com http://10.0.0.7 
elk.oldux.com 
[root@web01 conf.d]# curl -H Host:bs.oldxu.com http://10.0.0.7 
bs.oldux.com 
[root@web01 conf.d]# curl -H Host:bk.oldxu.com http://10.0.0.7 
bk.oldux.com

3.配置ﬁlebeat

filebeat.inputs:

- type: log
  enabled: true
  paths:
  - /var/log/nginx/elk.oldxu.com.log
    json.keys_under_root: true
    json.overwrite_keys: true
    tags: ["nginx-elk-host"]
- type: log
  enabled: true
  paths:
  - /var/log/nginx/bs.oldxu.com.log
    json.keys_under_root: true
    json.overwrite_keys: true
    tags: ["nginx-bs-host"]
- type: log
  enabled: true
  paths:
  - /var/log/nginx/bk.oldxu.com.log
    json.keys_under_root: true
    json.overwrite_keys: true
    tags: ["nginx-bk-host"]

- type: log
  enabled: true
  paths:
  - /var/log/nginx/error.log
    tags: ["nginx-error"]

output.elasticsearch:
  hosts: ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
  indices:
    - index: "nginx-elk-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "nginx-elk-host"


- index: "nginx-bs-access-%{[agent.version]}-%{+yyyy.MM.dd}"
  when.contains:
    tags: "nginx-bs-host"

- index: "nginx-bk-access-%{[agent.version]}-%{+yyyy.MM.dd}"
  when.contains:
    tags: "nginx-bk-host"

- index: "nginx-error-%{[agent.version]}-%{+yyyy.MM.dd}"
  when.contains:
    tags: "nginx-error"


setup.ilm.enabled: false
setup.template.name: nginx   #索引关联的模板名称
setup.template.pattern: nginx-*

Tomcat日志
访问日志 ---> json格式

#1.修改tomcat日志格式
[root@web02 soft]# yum install java -y 
[root@web02 soft]# vim tomcat/conf/server.xml

配置ﬁlebeat

filebeat.inputs:

- type: log
  enabled: true
  paths:
  - /soft/tomcat/logs/tomcat.oldxu.com.log.*.txt
    json.keys_under_root: true    #默认Flase，还会将json解析的日志存储至messages字段
    json.overwrite_keys: true     #覆盖默认的key，使用自定义json格式的key

output.elasticsearch:
  hosts: ["10.0.0.161:9200","10.0.0.162:9200"]
  index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"

setup.ilm.enabled: false
setup.template.name: tomcat   #索引关联的模板名称
setup.template.pattern: tomcat-*

错误日志 <--java

filebeat.inputs:

- type: log
  enabled: true
  paths:
  - /soft/tomcat/logs/tomcat.oldxu.com.log.*.txt
    json.keys_under_root: true    #默认Flase，还会将json解析的日志存储至messages字段
    json.overwrite_keys: true     #覆盖默认的key，使用自定义json格式的key
    tags: ["tomcat-access"]
- type: log
  enabled: true
  paths:
  - /soft/tomcat/logs/catalina.out
    multiline.pattern: '^\d{2}'   #匹配以2个数字开头的
    multiline.negate: true
    multiline.match: after
    multiline.max_lines: 10000    #默认最大合并行为500，可根据实际情况调整。
    tags: ["tomcat-error"]

output.elasticsearch:
  hosts: ["10.0.0.161:9200","10.0.0.162:9200"]
  indices:
    - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
      when.contains:
        tags: "tomcat-access"


- index: "tomcat-error-%{[agent.version]}-%{+yyyy.MM.dd}"
  when.contains:
    tags: "tomcat-error"


setup.ilm.enabled: false
setup.template.name: tomcat   #索引关联的模板名称
setup.template.pattern: tomcat-*

ELk-filebest

你可能感兴趣的:(ELk-filebest)