版本选择7.10.2,可配置中文版。
logstash-7.10.2、elasticsearch-7.10.2、kibana-7.10.2
下载完,解压,例如:tar -zxvf logstash-7.10.2-linux-x86_64.tar.gz
cd logstash-7.10.2/config
vim logstash-yjg.conf
#读
input {
# file 从项目文件读取
file {
path => ["/data/myProject/logs/*.log"]
#排除不想监听的文件
#exclude => "exclude.log"
#设置多长时间扫描目录,发现新文件
discover_interval => 1
#自定义日志区分类型
type => "ThisIsMyProjectType"
# beginning 从文件开始处读写
# end 文件结尾处读写
start_position => end
}
# nginx 日志收集
file {
path => ["/tmp/logs/*.log"]
discover_interval => 1
type => "nginx"
start_position => end
}
# http 方式收集,用于客户端上送消息
http{
id => "http_request"
port => 9062
codec => json
type => "http"
}
}
#过滤
filter {
if [type] == "http" {
# 把东八区时间赋值给新建的logdate 字段,logstash 不允许自建的特殊字段如带@标识的字段
ruby{
code => "event.set('logdate', (event.get('@timestamp').time.localtime).strftime('%Y-%m-%d %H:%M:%S'))"
}
} else if [type] == "nginx" {
grok {
match => ["message", "%{IPORHOST:host} - - \[%{HTTPDATE:logdate}\]"]
overwrite => [ "host"]
}
} else {
grok {
match => ["message", "%{TIMESTAMP_ISO8601:logdate}"]
}
}
date {
match => ["logdate", "YYYY-MM-dd HH:mm:ss.SSS"]
# 赋值给 @timestamp
target => "@timestamp"
}
mutate {
# 删除不需要的字段
remove_field => ["@version","tags","headers"]
}
}
#输出
output{
#输出到ES
elasticsearch{
# es地址
hosts=>["127.0.0.1:9200"]
# es的index名字,默认就是这个,可以更改
index => "yjg-logstash-%{+YYYY.MM.dd}"
}
#输出到控制台
stdout{
codec => rubydebug
}
# 输出到kafka
#kafka {}
# 输出到数据库
# jdbc {
# driver_jar_path => "/mysql-connector-java-5.1.48.jar"
# driver_class => "com.mysql.jdbc.Driver"
# connection_string => "jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useAffectedRows=true"
# username => "root"
# password => "123456"
# statement => ["INSERT INTO logstash_log_temp(level, ip, content, logdate) VALUES( ?, INET_ATON(?), ? ,?)", "[level]","[host]", "[message]", "[logdate]"]
#
#}
}
启动
# 启动logstash,并加载指定配置文件
./bin/logstash -f config/logstash-yjg.conf
注意:如果out 到数据库的话,因为插件不是默认安装,所以需要手动安装
# 官方拉取插件
./bin/logstash-plugin install logstash-output-jdbc
cd elasticsearch-7.10.2/config
vim elasticsearch.yml
path.data: /data/elasticsearch-7.10.2/data
path.logs: /data/elasticsearch-7.10.2/logs
network.host: 127.0.0.1
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"
注意:elaticsearch默认不能用root用户启动,否则会提示 java.lang.RuntimeException: can not run elasticsearch as root
# 为elaticsearch创建用户
groupadd elsearch
useradd elsearch -g elsearch -p elasticsearch
# 分配权限
chown -R elsearch:elsearch elasticsearch-7.10.2
# 切换用户到elsearch
su elsearch
启动
# 后台启动
./bin/elasticsearch &
# 验证启动是否成功
http://127.0.0.1::9200
{
"name" : "my-name",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "Px7djFasdfasdfa4eGTSbQ",
"version" : {
"number" : "7.10.2",
"build_flavor" : "default",
"build_type" : "tar",
"build_hash" : "747e1cc71def077253878a123123123123",
"build_date" : "2021-01-13T00:42:12.435326Z",
"build_snapshot" : false,
"lucene_version" : "8.7.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
cd kibana-7.10.2-linux-x86_64/config
vim kibana.yml
server.port: 5601
server.host: "127.0.0.1"
server.name: "my-kibana"
elasticsearch.hosts: ["http://127.0.0.1:9200"]
kibana.index: ".kibana"
# 设置中文(注意,如果是kibana-6*版本的话,需要下载python的服务,手动翻译,替换原文件,比较麻烦,故使用7版本)
i18n.locale: "zh-CN"
启动
# 启动kibana
./bin/kibana
注意:kibana 不支持root用户启动,否则提示:Kibana should not be run as root. Use --allow-root to continue。如果硬是要用root用户启动 就在后面加 --allow-root
# kibana 强制使用root用户启动
./bin/kibana --allow-root
注意:@timestamp 默认为UTC时间,比北京时间采集时间少8个小时。可以在kibana高级设置中更改
至此搭建全部结束,下期分享logstash采集和kibana查询规则