Linux localhost.localdomain 3.10.0-1160.6.1.el7.x86_64 #1 SMP Tue Nov 17 13:59:11 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux
安装 Logstash
- 安装 Java 环境
yum install -y java-1.8.0-openjdk
- 添加 repo：
/etc/yum.repos.d/elastic.repo
[elasticsearch]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md
- 安装 Logstash
yum --enablerepo=elasticsearch install logstash
- 验证安装
[root@localhost ~]# /usr/share/logstash/bin/logstash -e'input {stdin {}} output {stdout {}}'
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2020-11-26 11:28:49.727 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[INFO ] 2020-11-26 11:28:49.748 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[WARN ] 2020-11-26 11:28:50.266 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2020-11-26 11:28:50.287 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.8.13"}
[INFO ] 2020-11-26 11:28:50.324 [LogStash::Runner] agent - No persistent UUID file found. Generating new UUID {:uuid=>"db92df99-88f9-4134-9f91-14743bdb5d83", :path=>"/usr/share/logstash/data/uuid"}
[INFO ] 2020-11-26 11:28:58.501 [Converge PipelineAction::Create] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[INFO ] 2020-11-26 11:28:58.634 [Converge PipelineAction::Create] pipeline - Pipeline started successfully {:pipeline_id=>"main", :thread=>"#"}
The stdin plugin is now waiting for input:
[INFO ] 2020-11-26 11:28:58.710 [Ruby-0-Thread-2: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[INFO ] 2020-11-26 11:28:59.016 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
hello world
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"message" => "hello world",
"@timestamp" => 2020-11-26T03:29:13.910Z,
"@version" => "1",
"host" => "localhost.localdomain"
}
^C[WARN ] 2020-11-26 11:29:20.040 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2020-11-26 11:29:20.369 [[main]-pipeline-manager] pipeline - Pipeline has terminated {:pipeline_id=>"main", :thread=>"#"}
[INFO ] 2020-11-26 11:29:20.417 [LogStash::Runner] runner - Logstash shut down.
- 配置 Logstash：在
/etc/logstash/conf.d目录下创建远程日志配置文件remote_syslog.conf
input {
syslog {
port => "514"
type => "syslog"
}
}
filter {}
output {
elasticsearch {
hosts => "localhost:9200"
index => "syslog-%{+YYYY.MM.dd}"
}
}
安装 Elasticsearch
- 安装 Elasticsearch
yum --enablerepo=elasticsearch install elasticsearch
- 扩大系统最大文件数：
/etc/security/limits.d/20-nproc.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 4096
* hard nproc 4096
root soft nproc unlimited
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
- Daemon reload
systemctl daemon-reload
- 配置 MEMLOCK：
/usr/lib/systemd/system/elasticsearch.service
[Service]
LimitMEMLOCK=infinity
- 修改 elasticsearch 配置：
/etc/elasticsearch/elasticsearch.yml
bootstrap.memory_lock: true
http.port: 9200
- 根据系统资源情况修改 heap size：
/etc/elasticsearch/ jvm.options
-Xms4g
-Xmx4g
- 解决虚拟内存太低，增加最大字节处理长度：
/etc/sysctl.conf
vm.max_map_count=655360
# 查看激活
[root@localhost ~]# sysctl -p
vm.overcommit_memory = 1
vm.max_map_count = 655360
- 启动 elasticsearch
systemctl start Elasticsearch
- 访问
[root@localhost ~]# curl http://localhost:9200/
{
"name" : "l1_orbv",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "2K5T_E63Raqt0lGX8xls_A",
"version" : {
"number" : "6.8.13",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "be13c69",
"build_date" : "2020-10-16T09:09:46.555371Z",
"build_snapshot" : false,
"lucene_version" : "7.7.3",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
- 查看 index 状态
[root@localhost ~]# curl 'localhost:9200/_cat/indices?v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open .kibana_task_manager sp7Yi1R6Qj22xj_WGkEgbg 1 0 2 0 13.2kb 13.2kb
green open .monitoring-es-6-2020.11.26 _-coeXyrQQShTuIYYcwqLQ 1 0 6035 30 5mb 5mb
green open .monitoring-kibana-6-2020.11.26 94IU5EWpRhOakdHGNxLBGQ 1 0 594 0 492.9kb 492.9kb
yellow open syslog-2020.11.26 TiFIWTwzSwmC6E7zGKAtfQ 5 1 1 0 9.7kb 9.7kb
green open .kibana_1 RQ95lJkuRb-EYMoeU__g8A 1 0 20 2 93.2kb 93.2kb
green open kibana_sample_data_logs wZf5Vsj2QnKB_lfOiJy44Q 1 0 14005 0 11.6mb 11.6mb
安装 kibana
- 安装 kibana
yum --enablerepo=elasticsearch install kibana
- 配置
/etc/kibana/kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]
- 启动 kibana & logstash
systemctl start Kibana
systemctl start logstash
-
访问 kibana 并添加 index：
http://x.x.x.x:5601/
添加 index -
查看收到的日志
查看日志
References
https://gryzli.info/2019/02/15/installing-and-configuring-filebeat-on-centos-rhel/
https://www.cnblogs.com/xiaobaozi-95/p/9214307.html
https://blog.csdn.net/weixin_42385705/article/details/103342947?utm_medium=distribute.pc_relevant.none-task-blog-baidulandingword-2&spm=1001.2101.3001.4242