飞天使-elk搭建补充

es 集群创建密码

参考这篇博文进行设置：https://juejin.cn/post/7079955586330132487
最后的效果
#curl -XGET 'http://127.0.0.1:9200/_cat/nodes?pretty' -u elastic:gfsdfdsfesfes
172.16.100.6 25 51 0 0.04 0.32 0.22 cdfhilmrstw * node-3
172.16.100.5 25 51 2 0.05 0.22 0.15 cdfhilmrstw - node-2
172.16.100.4 21 51 0 0.10 0.34 0.23 cdfhilmrstw - node-1


es配置
cluster.name: k
node.name: node-1
path.data: /data/esdata
path.logs: /data/eslogs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["172.16.100.4", "172.16.100.5", "172.16.100.6"]
cluster.initial_master_nodes: ["node-1","node-2","node-3"]
node.master: true
node.data: true

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /data/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /data/elasticsearch/config/elastic-certificates.p12

kibana 配置文件以及和nginx配置

配置文件
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: 
server.basePath: "/elk"
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
["http://172.16.100.4:9200","http://172.16.100.5:9200","http://172.16.100.6:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "gLUAjdwadwadwwda"


server {
  listen       80 ;
  listen 443 ssl http2;
  server_name  elkfront.xxx.com;

  if ($server_port ~ 80){
        rewrite ^  https://$host/elk$request_uri? permanent;
  }
  ssl_certificate  /server/key/xxx.com.crt;
  ssl_certificate_key /server/key/xxx.com.key;
 


  location / {
    rewrite ^/$ /elk redirect;
  }


   location /elk/ {
       proxy_pass http://127.0.0.1:5601;
       proxy_redirect off;
       proxy_set_header Host $host:9091;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
       rewrite ^/elk/(.*)$ /$1 break;
   }
}

pm2 安装

centos7.9 版本 
以下是在 CentOS 7.9 上使用 nvm 安装 Node.js 的步骤：


安装 nvm：

bashCopy Code


curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash

在终端中重新加载 shell 配置或打开一个新的终端窗口：

bashCopy Code


source ~/.bashrc

安装适用于您的系统的 Node.js 版本。例如，您可以尝试安装 v14.17.6：

bashCopy Code


nvm install v14.17.6

使用 nvm 切换到安装的 Node.js 版本：

bashCopy Code


nvm use v14.17.6

确认 Node.js 版本已切换成功：

bashCopy Code


node -v

现在，您应该能够使用较新版本的 Node.js，而不会出现与 glibc 版本不兼容的错误。

npm install pm2 -g

pm2 结合pm2 进行脚本控制
#cat logstash_main.sh 
#!/bin/bash
/data/logstash_main/bin/logstash -f /data/logstash_main/config/logstash.conf  >> /data/logs/logstash_main.log  2>&1

pm2 start /data/scripts/logstash_main.sh


redis 端口监控
*/5 * * * *  /bin/bash /importredis.sh >/dev/null 2>&1 &
#cat importredis.sh 
#!/bin/bash
gg=`netstat -an |grep ":6379" |awk '$1 == "tcp" && $NF == "LISTEN" {print $0}'|wc -l`
if [[ ${gg}x = "0x" ]];then
    systemctl start redis  >/dev/null 2>&1 &
fi

定期清理索引以及告警

30 13 * * 7 sh /root/delete.sh >/dev/null 2>&1
delete.sh
curl -XDELETE 'http://127.0.0.1:9200/xx-jt*' -u elastic:gLcdsdfsdfesdfe


告警部分
#cat http_status.py 
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch
import sys
import requests

index=sys.argv[1] #要查询的索引
http_code=sys.argv[2] #要统计的状态码
limit=int(sys.argv[3])

def send_msg(status,count,index):  #上线通知功能
    token = "2104vfvdffvfdvsdfM" #
    chat_id = -73223443232 # 替换成你自己的chat_id
    try:
        ret = requests.post(f'https://api.telegram.org/bot{token}/sendMessage', json={"chat_id": chat_id,
        "text": "ELK通知:\n状态码: {status} \n数量: {count} \n索引: {index}".format(status=status,count=count,index=index)},proxies=None)
        print(r.json())
    except:
        print("error")
        pass

es = Elasticsearch(['http://10.0.0.4:9200','http://10.0.0.5:9200','http://10.0.0.6:9200'],http_auth=('elastic', 'gLUAjIJfesfesfsef'),)
query = {
    "query": {
        "bool": {
            "filter": [
                {"range": {"@timestamp": {"gte": "now-5m", "lte": "now"}}},
                {"bool": {
                    "should": [
                        {"term": {"status": http_code}},
                        #{"term": {"status": 502}},
                        #j{"term": {"status": 503}},
                        #{"term": {"status": 504}}
                    ],
                    "minimum_should_match": 1
                }}
            ]
        }
    }
}

result = es.search(index=index, body=query)

print(result['hits']['total'])
count=result['hits']['total'].get('value')
print(count)
if count>limit:
    print("出现异常记录\n")
    send_msg(http_code,count,index)  #上线通知功能

定时计划任务 


# xxxxx日志索引
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 500 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 502 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 503 20
*/1 * * * * python3 /data/shell/http_status.py xxxxx* 504 20

logstash 配置

 jvm.options 配置
-Xms2g
-Xmx2g


#cat logstash.conf 
input {
   # 从文件读取日志信息
   redis {
        host => "10.0.0.7"
        port => 6379
        password => "Rcfesfefesfesfes"  #如果没有密码不需要写这个参数
        key => "nginx1"
        data_type => "list"
        db => 0
    }
 }
 
filter {
    json {
       source => "message"
       remove_field => ["beat","offset","tags","prospector"] #移除字段，不需要采集
    }
    date {
      match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"] #匹配timestamp字段
      target => "@timestamp"  #将匹配到的数据写到@timestamp字段中
    }
}
 
output {
       if [filetype] == "a1_nginxjson" {
         elasticsearch {
            hosts => ["10.0.0.4:9200","10.0.0.5:9200","10.0.0.6:9200"]
            index => "jxxxxx1-%{+YYYY.MM.dd}"
            user => elastic
            password => 'gLUAjIJwnuHb1Rp21gWZ'
        }
       } else if [filetype] == "a2_nginxjson" {
         elasticsearch {
            hosts => ["10.0.0.4:9200","10.0.0.5:9200","10.0.0.6:9200"]
            index => "jxxxxx-%{+YYYY.MM.dd}"
            user => elastic
            password => 'fesfefesfadfsd'
        }
       } 
} 

filebeat 配置文件

#cat filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  backoff: "1s"
  tail_files: false
  paths:
    - /home/*.log
  fields:
    filetype: log_nginxjson
  fields_under_root: true
 
- type: log
  enabled: true
  backoff: "1s"
  tail_files: false
  paths:
    - /usr/local/nginx/logs/*.log
  fields:
    filetype: a1_nginxjson
  fields_under_root: true

output.redis:
  enabled: true
  hosts: ["10.0.0.7:6379"]
  password: Rcfesfefesfesfes
  key: nginx1
  db: 0

nginx 的日志索引

        log_format mainJson
                      '{"@timestamp":"$time_iso8601",'
                      '"host":"$hostname",'
                      '"server_ip":"$server_addr",'
                      '"http_x_forwarded_for":"$http_x_forwarded_for",'
                      '"domain":"$host",'
                      '"url":"$uri",'
                      '"referer":"$http_referer",'
                      '"args":"$args",'
                      '"upstreamtime":"$upstream_response_time",'
                      '"responsetime":"$request_time",'
                      '"request_method":"$request_method",'
                      '"status":"$status",'
                      '"size":"$body_bytes_sent",'
                      #'"request_body":"$request_body",'
                      '"request_length":"$request_length",'
                      '"protocol":"$server_protocol",'
                      '"upstreamhost":"$upstream_addr",'
                      '"file_dir":"$request_filename",'
                      '"http_user_agent":"$http_user_agent",'
                      '"remote_addr":"$remote_addr",'
                      '"client_ip": "$remote_addr",'
                      '"request_uri":"$request_uri",'
                      '"request_completion":"$request_completion"'
    '}';

参考网址:https://juejin.cn/post/7079955586330132487