目录
环境
【1】redis配置
【2】filebeat配置
【3】对接logstash配置
【4】验证
【5】安全配置:第一种:kibana-nginx访问控制
【6】第二种:在ES-主节点-配置TLS
【7】kibana配置密码
【8】logstash添加用户密码
es-01,kibana |
10.0.0.21 |
es-02 |
10.0.0.22 |
es-03 |
10.0.0.23 |
filebeat,nginx |
10.0.0.25 |
logstash |
10.0.0.26 |
redis |
10.0.0.27 |
[root@redis ~]# vim /etc/redis.conf
.....
bind 0.0.0.0
.....
requirepass 111
.....
[root@filebeat ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.redis:
hosts: ["10.0.0.27:6379"]
password: "111"
db: 0
timeout: 5
keys:
- key: "filebeat-nginx-access"
when.contains:
tags: "access"
- key: "filebeat-nginx-error"
when.contains:
tags: "error"
[root@filebeat ~]# systemctl restart filebeat.service
## 验证redis是否有数据写入,查看key的长度
[root@redis ~]# redis-cli
127.0.0.1:6379> auth 111
OK
127.0.0.1:6379> keys *
1) "filebeat-nginx-access"
2) "filebeat-nginx-error"
127.0.0.1:6379> select 0
OK
127.0.0.1:6379> LLEN filebeat-access
(integer) 19
127.0.0.1:6379> LLEN filebeat-error
(integer) 15
127.0.0.1:6379>
[root@logstash ~]# cat /etc/logstash/conf.d/file-redis-log.conf
input {
redis {
host => "10.0.0.27"
password => "111"
db => "0"
data_type => "list"
key => "filebeat-access"
}
redis {
host => "10.0.0.27"
password => "111"
db => "0"
data_type => "list"
key => "filebeat-error"
}
}
filter {
if "access" in [tags][0] {
grok {
match => {
"message" => "%{COMBINEDAPACHELOG}"
}
}
geoip {
source => "clientip"
}
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
useragent {
source => "agent"
target => "useragent"
}
mutate {
convert => ["bytes","integer"]
convert => ["response_time", "float"]
convert => ["upstream_response_time", "float"]
remove_field => ["message"]
add_field => { "target_index" => "app-logstash-nginx-access-%{+YYYY.MM.dd}" }
}
# 提取 referrer 具体的域名 /^"http/
if [referrer] =~ /^"http/ {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }
}
}
# 提取用户请求资源类型以及资源 ID 编号
if "test.com" in [referrer_host] {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:test_type}/%{NOTSPACE:test_res_id})?' }
}
}
}
else if "error" in [tags][0] {
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
mutate {
add_field => { "target_index" => "app-logstash-nginx-error-%{+YYYY.MM.dd}" }
}
}
}
output {
elasticsearch {
hosts => ["10.0.0.21:9200","10.0.0.22:9200","10.0.0.23:9200"]
index => "%{[target_index]}"
template_overwrite => true
}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/file-redis-log.conf -r
[root@es-01 ~]# yum -y install httpd-tools.x86_64
[root@es-01 ~]# htpasswd -c -b /etc/nginx/basic_passwd kibana 123123
Adding password for user kibana
[root@es-01 ~]# cat /etc/nginx/basic_passwd
kibana:$apr1$SASZLDcF$mHc2stQeQOCYXeapYhMj7/
[root@es-01 ~]# vim /etc/nginx/conf.d/kibana.conf
server {
listen 80;
server_name nginx.kibana.org;
location / {
proxy_pass http://127.0.0.1:5601$request_uri;
auth_basic "请输入用户名、密码";
auth_basic_user_file /etc/nginx/basic_passwd;
}
}
[root@es-01 ~]# systemctl start nginx
## 在主节点配置TLS
[root@es-01 ~]# /usr/share/elasticsearch/bin/elasticsearch-certutil \
> cert -out /etc/elasticsearch/elasticsearch-certificates.p12 -pass ""
[root@es-01 ~]# chmod 660 /etc/elasticsearch/elasticsearch-certificates.p12
## 编辑配置文件,添加以下内容,所有节点都需要添加
[root@es-01 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12
[root@es-02 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12
[root@es-03 ~]# vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elasticsearch-certificates.p12
xpack.security.transport.ssl.truststore.path: elasticsearch-certificates.p12
## 将TLS证书拷贝到所有节点
[root@es-01 ~]# scp -rp /etc/elasticsearch/elasticsearch-certificates.p12 [email protected]:/etc/elasticsearch/
[root@es-01 ~]# scp -rp /etc/elasticsearch/elasticsearch-certificates.p12 [email protected]:/etc/elasticsearch/
## 重新启动所有节点
[root@es-01 ~]# systemctl restart elasticsearch
## 主节点运行后,集群配置密码,auto随机密码,interactive手动
[root@es-01 ~]# /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
The passwords will be randomly generated and printed to the console.
Please confirm that you would like to continue [y/N]y
Changed password for user apm_system
PASSWORD apm_system = IFYVDU9pa0BaYqn8bFdi
Changed password for user kibana
PASSWORD kibana = 3S32IGJTj9bpgIy8q4bP
Changed password for user logstash_system
PASSWORD logstash_system = EbaDFZhIMqNMgtMWaBFY
Changed password for user beats_system
PASSWORD beats_system = hFivaRmv8BUGxr11vkRM
Changed password for user remote_monitoring_user
PASSWORD remote_monitoring_user = 68xmYgepf2L40Zr5HPLI
Changed password for user elastic
PASSWORD elastic = PpCC9KV8FnoDepE7DYZU
[root@es-01 ~]#
[root@es-01 ~]# vim /etc/kibana/kibana.yml
......
elasticsearch.username: "kibana"
elasticsearch.password: "vERKRjujB5kfukYCT77w"
[root@es-01 ~]# systemctl restart kibana
[root@logstash ~]# cat /etc/logstash/conf.d/file-redis-log.conf
input {
redis {
host => "10.0.0.27"
password => "111"
db => "0"
data_type => "list"
key => "filebeat-access"
}
redis {
host => "10.0.0.27"
password => "111"
db => "0"
data_type => "list"
key => "filebeat-error"
}
}
filter {
if "access" in [tags][0] {
grok {
match => {
"message" => "%{COMBINEDAPACHELOG}"
}
}
geoip {
source => "clientip"
}
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
useragent {
source => "useragent"
target => "useragent"
}
mutate {
convert => ["bytes","integer"]
convert => ["response_time", "float"]
convert => ["upstream_response_time", "float"]
remove_field => ["message"]
add_field => { "target_index" => "app-logstash-nginx-access-%{+YYYY.MM.dd}" }
}
# 提取 referrer 具体的域名 /^"http/
if [referrer] =~ /^"http/ {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }
}
}
# 提取用户请求资源类型以及资源 ID 编号
if "test.com" in [referrer_host] {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:test_type}/%{NOTSPACE:test_res_id})?' }
}
}
}
else if "error" in [tags][0] {
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
mutate {
add_field => { "target_index" => "app-logstash-nginx-error-%{+YYYY.MM.dd}" }
}
}
}
output {
elasticsearch {
hosts => ["10.0.0.21:9200","10.0.0.22:9200","10.0.0.23:9200"]
user => "logstash_push_es"
password => "111111"
index => "%{[target_index]}"
template_overwrite => true
}
stdout {
codec => rubydebug
}
}