filebeat-modules收集nginx日志步骤

###########################################################

filebeat modules 自定义索引和视图：

1.为了不影响实验，建议删除所有其他的索引

systemctl stop elasticsearch
systemctl stop kibana
rm -rf /data/elasticsearch/*
rm -rf /var/lib/kibana/*
systemctl start elasticsearch
systemctl start kibana

2.修改nginx配置文件

sed -i 's#json#main#g' /etc/nginx/conf.d/bbs.conf

3.清空nginx日志

> /var/log/nginx/bbs_access.log

4.重启nginx

systemctl restart nginx

5.修改filebeat配置文件:

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
setup.kibana:
  host: "10.0.0.51:5601"
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  indices:
    - index: "nginx_bbs_access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        source: "/var/log/nginx/bbs_access.log"
    - index: "nginx_error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        fileset.name: "error"
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true

6.激活nginx模块报错

filebeat modules enable nginx

7.安装nginx modules插件

/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-geoip-6.6.0.zip
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-user-agent-6.6.0.zip 

8.重启es

systemctl restart elasticsearch

9.修改模块配置

[root@db01 ~]# egrep -v "#|^$" /etc/filebeat/modules.d/nginx.yml 
- module: nginx
  access:
    enabled: true
    var.paths: ["/var/log/nginx/bbs_access.log"]
  error:
    enabled: true
    var.paths: ["/var/log/nginx/error.log"]

10.备份删除不必要的视图文件并导入到kibana

cp -a /usr/share/filebeat/kibana /root
cd /usr/share/filebeat/kibana/6/dashboard
find . -type f ! -name "*nginx*"|xargs rm -rf
rm -rf  ml-nginx-*
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-logs.json 
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-overview.json
cd index-pattern/
sed -i 's#filebeat\-\*#nginx\_\*#g' filebeat.json
filebeat setup --dashboards -E setup.dashboards.directory=/root/kibana/
rm -rf /var/lib/kibana/*
systemctl restart kibana
#########################################################################