官网：
https://www.elastic.co/products
版本：
- Elasticsearch 2.3.4 ： 日志索引、存储、查询
- Logstash 2.3.4 ： 日志收集、过滤、转发
- Kibana 4.5.3 ： 可视化查询 Elasticsearch 的数据
- Beats Filebeat 1.2.3 ： 定时获取增量日志并转发给 Logstash
Topbeat 1.2.3 ： 定期收集系统信息并转发给 Logstash
前提：
设置hostname（做SSL证书需要）、安装Java（Elasticsearch和Logstash需要）
(1)导入GPG key
rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
(2)添加Yum仓库
# vi /etc/yum.repos.d/elasticsearch.repo [elasticsearch-2.x] name=Elasticsearch repository for 2.x packages baseurl=https://packages.elastic.co/elasticsearch/2.x/centos gpgcheck=1 gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch enabled=1 # vi /etc/yum.repos.d/kibana.repo [kibana-4.5] name=Kibana repository for 4.5.x packages baseurl=http://packages.elastic.co/kibana/4.5/centos gpgcheck=1 gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch enabled=1 # vi /etc/yum.repos.d/logstash.repo [logstash-2.3] name=Logstash repository for 2.3.x packages baseurl=https://packages.elastic.co/logstash/2.3/centos gpgcheck=1 gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch enabled=1 # vi /etc/yum.repos.d/beats.repo [beats] name=Elastic Beats Repository baseurl=https://packages.elastic.co/beats/yum/el/$basearch enabled=1 gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch gpgcheck=1
(3)安装Elasticsearch
# yum -y install elasticsearch # chkconfig --add elasticsearch # chkconfig elasticsearch on # rpm -qc elasticsearch # vi /etc/elasticsearch/elasticsearch.yml network.host: localhost # service elasticsearch start # service elasticsearch status # netstat -nltp | grep -E '9200|9300' # curl -X GET 'http://localhost:9200'
确认Elasticsearch的索引
# curl http://localhost:9200/_cat/indices
(4)安装Kibana
# yum -y install kibana # chkconfig --add kibana # chkconfig kibana on # rpm -qc kibana # vi /opt/kibana/config/kibana.yml server.host: "localhost" # service kibana start # service kibana status # netstat -nltp | grep 5601 # curl -X GET 'http://localhost:5601'
Kibana Dashboards
# cd /usr/local/src/ # curl -L -O https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip # unzip beats-dashboards-1.1.0.zip # cd beats-dashboards-1.1.0 # ./load.sh
(5)安装Logstash
# yum -y install logstash
# chkconfig --add logstash
# chkconfig logstash on
# hostname
# cd /etc/pki/tls
# openssl req -subj '/CN=ELK_server_fqdn/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt
# vi /etc/logstash/conf.d/02-beats-input.conf
input {
beats {
port => 5043
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
# vi /etc/logstash/conf.d/10-syslog-filter.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
# vi /etc/logstash/conf.d/30-elasticsearch-output.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
# service logstash configtest
# service logstash start
# service logstash status
(6)安装Filebeat
# cp /tmp/logstash-forwarder.crt /etc/pki/tls/certs/logstash-forwarder.crt # cd ~ # curl -O https://gist.githubusercontent.com/thisismitch/3429023e8438cc25b86c/raw/d8c479e2a1adcea8b1fe86570e42abab0f10f364/filebeat-index-template.json # curl -X PUT 'http://localhost:9200/_template/filebeat?pretty' [email protected] # yum -y install filebeat # chkconfig --add filebeat # chkconfig filebeat on # vi /etc/filebeat/filebeat.yml # egrep -v '^$|^#|^\s+#' /etc/filebeat/filebeat.yml filebeat: prospectors: - paths: - /var/log/*.log input_type: log document_type: syslog registry_file: /var/lib/filebeat/registry output: logstash: hosts: ["localhost:5043"] tls: certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"] shipper: logging: files: rotateeverybytes: 10485760 # = 10MB # service filebeat start # service filebeat status # curl -XGET 'http://localhost:9200/filebeat-*/_search?pretty'
(7)安装Topbeat
# cd ~ # curl -O https://raw.githubusercontent.com/elastic/topbeat/master/etc/topbeat.template.json # curl -XPUT 'http://localhost:9200/_template/topbeat' [email protected] # yum -y install topbeat # vi /etc/topbeat/topbeat.yml # egrep -v '^$|^#|^\s+#' /etc/topbeat/topbeat.yml output: logstash: hosts: ["localhost:5043"] tls: certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"] # service topbeat start # service topbeat status # curl -XGET 'http://localhost:9200/topbeat-*/_search?pretty'
(8)收集Nginx日志
# mkdir -p /opt/logstash/patterns
# chown logstash: /opt/logstash/patterns
# vi /opt/logstash/patterns/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent}
# chown logstash: /opt/logstash/patterns/nginx
# vi /etc/filebeat/filebeat.yml
-
paths:
- /var/log/nginx/access.log
document_type: nginx-access
# service filebeat restart
# vi /etc/logstash/conf.d/11-nginx-filter.conf
filter {
if [type] == "nginx-access" {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
}
}
# service logstash restart
(9)收集Apache日志
# vi /etc/filebeat/filebeat.yml
-
paths:
- /var/log/apache2/access.log
document_type: apache-access
# service filebeat restart
# vi /etc/logstash/conf.d/12-apache.conf
filter {
if [type] == "apache-access" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
}
# service logstash restart
参考：
https://www.elastic.co/guide/index.html
https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-centos-7