Logstash安装部署

目录

一、环境准备

二、安装部署

2.1 下载安装包到指定文件夹，并解压

2.2 复制证书文件

2.3 编辑配置文件

2.4 启动服务

---

一、环境准备

    部署模式：单节点部署。

    官网地址：Elasticsearch 平台 — 大规模查找实时答案 | Elastic

    注意事项：

        1.  部署及安装所用的用户不能是root

192.168.122.119	Centos7.6	node1.vteamcloud.com

二、安装部署

2.1 下载安装包到指定文件夹，并解压

# 切换到非root用户，这里用的elasticsearch
su - elasticsearch
# 进入安装目录
cd /opt/module
# 解压安装包
tar xf logstash-8.11.0.tar.gz
# 给文件赋权
chown -R elasticsearch:elasticsearch /opt/module/logstash-8.11.0

2.2 复制证书文件

# 进入配置文件目录
cd logstash-8.11.0/config
# 创建证书文件夹
mkdir certs
# 将es的证书文件复制到certs文件夹下
cp /opt/module/elasticsearch-8.11.0/config/certs/http.p12 certs/

2.3 编辑配置文件

vim logstash.conf
 
# 从redis里面拿日志数据，这里配置的也是哨兵集群的1主2从共三个节点。
input {
  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-dev-log" #监听的键值
        host => "192.168.122.227" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-dev239"]
  }
  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-dev-log" #监听的键值
        host => "192.168.122.237" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-dev239"]
  }
  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-dev-log" #监听的键值
        host => "192.168.122.238" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-dev239"]
  }

  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-test-log" #监听的键值
        host => "192.168.122.227" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-test251"]

  }
  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-test-log" #监听的键值
        host => "192.168.122.237" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-test251"]
  }
  redis {
        batch_count => 1 #返回的事件数量，此属性仅在list模式下起作用。
        data_type => "list" #logstash redis插件工作方式
        key => "ipu-cbs-server-test-log" #监听的键值
        host => "192.168.122.238" #redis地址
        port => 6379 #redis端口号
        password => "Redis@123456" #如果有安全认证，此项为密码
        db => 0 #redis数据库的编号
        threads => 1 #启用线程数量
        tags => ["ipu-cbs-server-test251"]
  }


}
 
 
filter {
 
  # 去除message中日志颜色的转义符
  mutate {
    gsub => [
      "message", "\u001b\[32m", "",
      "message", "\u001b\[34m", "",
      "message", "\u001b\[35m", "",
      "message", "\u001b\[36m", "",
      "message", "\u001b\[0;39m", ""
    ]
  }
 
  grok {
   # match => { "message" => "%{DATESTAMP:logdate}" }
   # 将yy-MM-dd HH:mm:ss.SSS 格式的日期时间赋值为logdate
     match => { "message" => "%{TIMESTAMP_ISO8601:logdate} %{GREEDYDATA:log_message}" }
  }
 
  # 将logdate的值赋值给@timestamp
  date {
    match => [ "logdate", "YY-MM-dd HH:mm:ss.SSS" ]
    target => "@timestamp"
    timezone =>"+00:00"
  }
 
  mutate {
  #  add_field => { "offset" => "%{[log][offset]}"}
  #  add_field => { "logDateTime" => "%{logdate}"}
  #  删除不用的字段
    replace => { "message" => "%{log_message}" }
    remove_field => ["event","input","host","ecs","log","@version","agent","logdate","log_message"]
  }
 
# 将logDateTime转为日期类型
#  date {
#    match => ["logDateTime", "yy-MM-dd HH:mm:ss.SSS"]
#    target => "logDateTime"
#  }
}
 

output {

 if "ipu-cbs-server-test251" in [tags] {
  elasticsearch {
    hosts => ["https://192.168.122.118:9200","https://192.168.122.119:9200","https://192.168.122.120:9200"]
    index => "ipu-cbs-server-test"
 #   ssl => true
#    cacert => "/opt/module/logstash-8.11.0/config/certs/elasticsearch-ca.pem"   
    user => "elastic"
    password => "elastic"
    ssl_certificate_verification => true
    truststore => "/opt/module/logstash-8.11.0/config/certs/http.p12"
    truststore_password => "123456"
  }      
        
 } else if "ipu-cbs-server-dev239" in [tags] {
   elasticsearch {
    hosts => ["https://192.168.122.118:9200","https://192.168.122.119:9200","https://192.168.122.120:9200"]
    index => "ipu-cbs-server-dev"
 #   ssl => true
 #   #    cacert => "/opt/module/logstash-8.11.0/config/certs/elasticsearch-ca.pem"   
    user => "elastic"
    password => "elastic"
    ssl_certificate_verification => true
    truststore => "/opt/module/logstash-8.11.0/config/certs/http.p12"
    truststore_password => "123456" 
   }
 }
}

2.4 启动服务

# 编写启动命令文件
echo "nohup bin/logstash -f config/logstash.conf  > ./log/logstash.log 2>&1 &" > start.sh
# 赋予文件权限
chmod a+x start.sh
# 启动服务
./start.sh
# 查看日志
tail -200f /opt/module/logstash-8.11.0/log/logstash.log