centos7--搭建部署ELK服务

一，安装elasticsearch服务

1，在主机A上安装jdk，elasticsearch

[root@localhost ~] rpm -ivh jdk-8u131-linux-x64_.rpm 
准备中...                          ################################# [100%]
正在升级/安装...
   1:jdk1.8.0_131-2000:1.8.0_131-fcs  ################################# [100%]
Unpacking JAR files...
	tools.jar...
	plugin.jar...
	javaws.jar...
	deploy.jar...
	rt.jar...
	jsse.jar...
	charsets.jar...
	localedata.jar...
[root@localhost ~] java -version #查看安装成功，会生成版本号
openjdk version "1.8.0_161"
OpenJDK Runtime Environment (build 1.8.0_161-b14)
OpenJDK 64-Bit Server VM (build 25.161-b14, mixed mode)
[root@localhost ~] 

[root@localhost ~] rpm -ivh elasticsearch-6.6.2.rpm 
警告：elasticsearch-6.6.2.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中...                          ################################# [100%]
Creating elasticsearch group... OK
Creating elasticsearch user... OK
正在升级/安装...
   1:elasticsearch-0:6.6.2-1          ################################# [100%]
### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd
 sudo systemctl daemon-reload
 sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
 sudo systemctl start elasticsearch.service
Created elasticsearch keystore in /etc/elasticsearch
[root@localhost ~] 

2，修改elasticsearch配置文件

[root@localhost ~] vim /etc/elasticsearch/elasticsearch.yml 

# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: wg001  #定义集群名称
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
node.name: node-1 #定义节点名称
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
network.host: 192.168.59.142 # 设置主机ip
#
# Set a custom port for HTTP:
#
http.port: 9200 # ---定义监听端口
#
# For more information, consult the network module documentation.

3，添加开机自启，启动elasticsearch，查看端口和服务是否启动

[root@localhost ~] systemctl enable elasticsearch #添加开机自启
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.

[root@localhost ~] systemctl start elasticsearch #启动服务

[root@localhost ~] netstat -nuptl|grep 9200 #查看端口
tcp6       0      0 192.168.59.142:9200     :::*                    LISTEN      4766/java           

[root@localhost ~] curl -X GET http://192.168.59.142:9200 #查看节点信息
{
  "name" : "node-1",
  "cluster_name" : "wg001",
  "cluster_uuid" : "0L-ytIjaTfCwDDMVhanLzg",
  "version" : {
    "number" : "6.6.2",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "3bd3e59",
    "build_date" : "2019-03-06T15:16:26.864148Z",
    "build_snapshot" : false,
    "lucene_version" : "7.6.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}
[root@localhost ~]# 

查看日志

[root@localhost ~] tailf /var/log/elasticsearch/wg001.log #查看日志状态

二，安装logstash服务

4，在主机B上安装jdk和logstash

[root@localhost ~] rpm -ivh jdk-8u131-linux-x64_.rpm 
准备中...                          ################################# [100%]
正在升级/安装...
   1:jdk1.8.0_131-2000:1.8.0_131-fcs  ################################# [100%]
Unpacking JAR files...
	tools.jar...
	plugin.jar...
	javaws.jar...
	deploy.jar...
	rt.jar...
	jsse.jar...
	charsets.jar...
	localedata.jar...
[root@localhost ~]# 

[root@localhost ~] rpm -ivh logstash-6.6.0.rpm 
警告：logstash-6.6.0.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中...                          ################################# [100%]
正在升级/安装...
   1:logstash-1:6.6.0-1               ################################# [100%]
Using provided startup.options file: /etc/logstash/startup.options
Successfully created system startup script for Logstash
[root@localhost ~]# 

5，编写配置messages.conf文件

[root@localhost ~] vim /etc/logstash/conf.d/messages.conf

input { #定义进入模块
        file {
                path => "/var/log/messages" #日志路径
                type => "msg-log" #日志类型
                start_position => "beginning" #生成过程
        }
}
output { #
        elasticsearch {

                hosts => "192.168.59.142:9200" # 端口
                index => "msg_log-%{+YYYY.MM.dd}" # 索引
        }
}

6，启动logstash，检查端口，查看日志

[root@localhost ~] systemctl start logstash

[root@localhost ~] netstat -nuptl|grep 9600
tcp6       0      0 127.0.0.1:9600          :::*                    LISTEN      15053/java          
[root@localhost ~]#
[root@localhost ~] tailf /var/log/logstash/logstash-plain.log 

[2020-03-30T15:51:11,772][INFO ][logstash.inputs.file     ] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/var/lib/logstash/plugins/inputs/file/.sincedb_452905a167cf4509fd08acb964fdb20c", :path=>["/var/log/messages"]}

在主机A上验证

[root@localhost ~] tailf /var/log/elasticsearch/wg001.log 
#有这一条输出就OK了
[2020-03-30T15:50:16,989][INFO ][o.e.c.m.MetaDataMappingService] [node-1] [msg_log-2020.03.30/t_vmeYMrSXycGXjg0gN_kA] create_mapping [doc]

一般会出现这个报错

[2020-03-30T15:49:40,715][WARN ][filewatch.tailmode.handlers.createinitial] failed to open /var/log/messages: #, ["org/jruby/RubyFile.java:366:in `initialize'", "org/jruby/RubyIO.java:1154:in `open'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-input-file-4.1.9/lib/filewatch/watched_file.rb:198:in `open'"]

缺少权限，添加此命令 chmod 777 /var/log/ -R 之后重启logstash就会解决

三，安装部署kibana

7，在主机A上安装kibana，修改配置文件

[root@localhost ~] rpm -ivh kibana-6.6.2-x86_64.rpm 
警告：kibana-6.6.2-x86_64.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中...                          ################################# [100%]
正在升级/安装...
   1:kibana-6.6.2-1                   ################################# [100%]
[root@localhost ~]# 

[root@localhost ~] vim /etc/kibana/kibana.yml

# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601 #定义端口号

# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "192.168.59.142" #服务ip

# The URLs of the Elasticsearch instances to use for all your queries.
elasticsearch.hosts: ["http://192.168.59.142:9200"] 

启动，查看端口是否开启

[root@localhost ~] systemctl start kibana
[root@localhost ~] netstat -nuptl|grep 5601
tcp        0      0 192.168.59.142:5601     0.0.0.0:*               LISTEN      8559/node           
[root@localhost ~]#

8，在浏览器web端验证


创建索引

四，编写多个logstash

9，修改管道的配置，定义多个管道，区分日志的路径和目录

[root@localhost ~] vim /etc/logstash/pipelines.yml

- pipeline.id: msg
  path.config: "/etc/logstash/conf.d/messages.conf"
- pipeline.id: sec
  path.config: "/etc/logstash/conf.d/secure.conf"

在主机B上，编写logstash多个配置文件

[root@localhost ~] vim /etc/logstash/conf.d/secure.conf

input {
        file {
                path => "/var/log/secure"
                type => "sec-log"
                start_position => "beginning"
        }
}
output {
        elasticsearch {

                hosts => "192.168.59.142:9200"
                index => "sec_log-%{+YYYY.MM.dd}"
        }
}

10，重启logstash，查看logstash

[root@localhost ~] systemctl restart logstash

[root@localhost ~] tailf /var/log/logstash/logstash-plain.log 

[2020-03-30T18:46:44,320][INFO ][logstash.inputs.file     ] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/var/lib/logstash/plugins/inputs/file/.sincedb_730aea1d074d4636ec2eacfacc10f882", :path=>["/var/log/secure"]}
[2020-03-30T18:46:44,329][INFO ][logstash.inputs.file     ] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/var/lib/logstash/plugins/inputs/file/.sincedb_452905a167cf4509fd08acb964fdb20c", :path=>["/var/log/messages"]}

11，在浏览器web端，创建新索引