EFK集群模式搭建文档

拓扑图介绍：

192.168.43.99        elk-master

192.168.43.100      elk-slave01

192.168.43.101      elk-slave02

版本信息：

OS   : CentOS Linux release 7.7.1908 (Core)

EFK : ELASTICSEARCH-7.8.0-LINUX-X86_64.TAR.GZ 

            FILEBEAT-7.8.0-LINUX-X86_64.TAR.GZ 

            KIBANA-7.8.0-LINUX-X86_64.TAR.GZ

备注：

1.请尽量保持各个组件的版本一致性，这个官方是有具体说明，最省事的做法是保持版本一致性

2.EFK7.3以后的版本支持用户及角色管理，尽量选用版本高一点的稳定的版本

3.官方下载地址： https://www.elastic.co/cn/downloads/past-releases#

---

一、系统配置

1.调整参数

[root@localhost ~]# vim /etc/security/limits.conf

[root@localhost ~]# tail -n 2  /etc/security/limits.conf

* soft nofile 655360

* hard nofile 655360

[root@localhost ~]# tail -n 1 /etc/sysctl.conf

vm.max_map_count=655360

2.修改/etc/hosts

[root@localhost ~]# tail -n 3 /etc/hosts

192.168.43.99 elk-master

192.168.43.100 elk-slave01

192.168.43.101 elk-slave02

3.关闭firewall和selinux

[root@localhost ~]# systemctl disable firewalld

Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.

Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

[root@localhost ~]# systemctl stop firewalld

[root@localhost ~]# vim /etc/selinux/config

[root@localhost ~]# grep 'SELINUX' /etc/selinux/config

# SELINUX= can take one of these three values:

SELINUX=disabled

# SELINUXTYPE= can take one of three values:

SELINUXTYPE=targeted

4.安装openjdk1.8

[root@localhost ~]#

[root@localhost ~]# java -version

openjdk version "1.8.0_262"

OpenJDK Runtime Environment (build 1.8.0_262-b10)

OpenJDK 64-Bit Server VM (build 25.262-b10, mixed mode)

5.安装ntpd

[root@elk-master ~]# yum install ntp -y

[root@elk-master ~]# vim /etc/ntp.conf

[root@elk-master ~]# grep '^server' /etc/ntp.conf

server ntp.aliyun.com

[root@elk-master ~]# systemctl enable ntpd

Created symlink from /etc/systemd/system/multi-user.target.wants/ntpd.service to /usr/lib/systemd/system/ntpd.service.

[root@elk-master ~]# ntpdate ntp.aliyun.com

26 Aug 10:35:24 ntpdate[1442]: step time server 203.107.6.88 offset 0.708313 sec

[root@elk-master ~]# date

Wed Aug 26 10:35:34 CST 2020

[root@elk-master ~]# systemctl start ntpd

[root@elk-master ~]#

6.添加elsearch用户

[root@elk-master opt]# useradd elsearch

[root@elk-master opt]# tail -n 2 /etc/passwd

ntp:x:38:38::/etc/ntp:/sbin/nologin

elsearch:x:1000:1000::/home/elsearch:/bin/bash

[root@elk-master opt]#

---

二、ES组件配置

1.上传文件

[root@elk-master opt]# ll

total 666108

-rw-r--r-- 1 root root 319112561 Aug 25 22:05 elasticsearch-7.8.0-linux-x86_64.tar.gz

-rw-r--r-- 1 root root 334236568 Aug 25 22:05 kibana-7.8.0-linux-x86_64.tar.gz

[root@elk-master opt]# tar -xf elasticsearch-7.8.0-linux-x86_64.tar.gz

[root@elk-master opt]# tar -xf kibana-7.8.0-linux-x86_64.tar.gz

[root@elk-master opt]# ll

total 666108

drwxr-xr-x  9 root root      155 Jun 15 03:38 elasticsearch-7.8.0

-rw-r--r--  1 root root 319112561 Aug 25 22:05 elasticsearch-7.8.0-linux-x86_64.tar.gz

drwxr-xr-x 13 root root      266 Aug 25 22:11 kibana-7.8.0-linux-x86_64

-rw-r--r--  1 root root 334236568 Aug 25 22:05 kibana-7.8.0-linux-x86_64.tar.gz

[root@elk-master opt]#

2.编辑配置文件

master配置文件

[root@elk-master config]# cat elasticsearch.yml

cluster.name: es-cluster

node.name: elk-master

path.data: /opt/elasticsearch-7.8.1/data

path.logs: /opt/elasticsearch-7.8.1/logs

network.host: 0.0.0.0

http.port: 9200

node.data: true

node.master: true

discovery.seed_hosts: ["elk-master", "elk-slave01","elk-slave02"]

cluster.initial_master_nodes: ["elk-master"]

http.cors.enabled:  true

http.cors.allow-origin: "*"

slave01配置文件

[root@elk-slave01 config]# cat elasticsearch.yml

cluster.name: es-cluster

node.name: elk-slave01

path.data: /opt/elasticsearch-7.8.1/data

path.logs: /opt/elasticsearch-7.8.1/logs

network.host: 0.0.0.0

http.port: 9200

node.data: true

node.master: true

discovery.seed_hosts: ["elk-master", "elk-slave01","elk-slave02"]

cluster.initial_master_nodes: ["elk-master"]

http.cors.enabled:  true

http.cors.allow-origin: "*"

slave02配置文件

[root@elk-slave02 config]# cat elasticsearch.yml

cluster.name: es-cluster

node.name: elk-slave02

path.data: /opt/elasticsearch-7.8.1/data

path.logs: /opt/elasticsearch-7.8.1/logs

network.host: 0.0.0.0

http.port: 9200

node.data: true

node.master: true

discovery.seed_hosts: ["elk-master", "elk-slave01","elk-slave02"]

cluster.initial_master_nodes: ["elk-master"]

http.cors.enabled:  true

http.cors.allow-origin: "*"

3.修改属主信息

[root@elk-master bin]# chown -R elsearch.elsearch /opt/elasticsearch-7.8.1

[root@elk-slave01 opt]# cd /opt/elasticsearch-7.8.1/bin/

4.开启es

[root@elk-slave01 bin]# su elsearch

[elsearch@elk-slave01 bin]$ ./elasticsearch

5.测试

[root@elk-master ~]# curl http://192.168.43.99:9200/

{

  "name" : "elk-master",

  "cluster_name" : "es-cluster",

  "cluster_uuid" : "fhMc4s8nTdiRTHdOlVAsCQ",

  "version" : {

    "number" : "7.8.1",

    "build_flavor" : "default",

    "build_type" : "tar",

    "build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",

    "build_date" : "2020-07-21T16:40:44.668009Z",

    "build_snapshot" : false,

    "lucene_version" : "8.5.1",

    "minimum_wire_compatibility_version" : "6.8.0",

    "minimum_index_compatibility_version" : "6.0.0-beta1"

  },

  "tagline" : "You Know, for Search"

}

[root@elk-master ~]# curl http://192.168.43.100:9200/

{

  "name" : "elk-slave01",

  "cluster_name" : "es-cluster",

  "cluster_uuid" : "fhMc4s8nTdiRTHdOlVAsCQ",

  "version" : {

    "number" : "7.8.1",

    "build_flavor" : "default",

    "build_type" : "tar",

    "build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",

    "build_date" : "2020-07-21T16:40:44.668009Z",

    "build_snapshot" : false,

    "lucene_version" : "8.5.1",

    "minimum_wire_compatibility_version" : "6.8.0",

    "minimum_index_compatibility_version" : "6.0.0-beta1"

  },

  "tagline" : "You Know, for Search"

}

[root@elk-master ~]# curl http://192.168.43.101:9200/

{

  "name" : "elk-slave02",

  "cluster_name" : "es-cluster",

  "cluster_uuid" : "fhMc4s8nTdiRTHdOlVAsCQ",

  "version" : {

    "number" : "7.8.1",

    "build_flavor" : "default",

    "build_type" : "tar",

    "build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",

    "build_date" : "2020-07-21T16:40:44.668009Z",

    "build_snapshot" : false,

    "lucene_version" : "8.5.1",

    "minimum_wire_compatibility_version" : "6.8.0",

    "minimum_index_compatibility_version" : "6.0.0-beta1"

  },

  "tagline" : "You Know, for Search"

}

[root@elk-master ~]#

[root@elk-master ~]# curl http://192.168.43.99:9200/_cat/nodes?pretty

192.168.43.99  22 97 0 0.00 0.04 0.10 dilmrt * elk-master

192.168.43.100 53 96 0 0.01 0.04 0.07 dilmrt - elk-slave01

192.168.43.101 10 96 0 0.03 0.06 0.08 dilmrt - elk-slave02

[root@elk-master ~]#

---

三、Kibana的安装配置

1.修改配置文件

配置机器：elk-master

[root@elk-master opt]# chown -R elsearch.elsearch kibana-7.8.1-linux-x86_64

[root@elk-master opt]# ll

total 668376

drwxr-xr-x 10 elsearch elsearch      167 Aug 26 11:05 elasticsearch-7.8.1

-rw-r--r--  1 root    root    318334518 Aug 26 10:39 elasticsearch-7.8.1-linux-x86_64.tar.gz

-rw-r--r--  1 root    root      28557354 Aug 26 14:39 filebeat-7.8.1-x86_64.rpm

drwxr-xr-x 13 elsearch elsearch      266 Aug 26 10:42 kibana-7.8.1-linux-x86_64

-rw-r--r--  1 root    root    337517217 Aug 26 10:39 kibana-7.8.1-linux-x86_64.tar.gz

[root@elk-master opt]# cd kibana-7.8.1-linux-x86_64/config/

[root@elk-master config]# vim kibana.yml

[root@elk-master config]# grep -v '^#' kibana.yml | grep -v '^$'

server.port: 5601

server.host: "192.168.43.99"

elasticsearch.hosts: ["http://192.168.43.99:9200"]

[root@elk-master config]#

2.启动kibana

[root@elk-master bin]# su elsearch

[elsearch@elk-master bin]$ ./kibana

[root@elk-master bin]# ps -ef  | grep node

elsearch  2624  2609  1 14:32 pts/1    00:06:00 ./../node/bin/node ./../src/cli

root      3099  2751  0 20:24 pts/2    00:00:00 grep --color=auto node

[root@elk-master bin]# netstat -anultp | grep 5601

tcp        0      0 192.168.43.99:5601      0.0.0.0:*              LISTEN      2624/./../node/bin/

tcp        0      0 192.168.43.99:5601      192.168.43.201:55089    ESTABLISHED 2624/./../node/bin/

tcp        0      0 192.168.43.99:5601      192.168.43.201:55074    ESTABLISHED 2624/./../node/bin/

tcp        0      0 192.168.43.99:5601      192.168.43.201:55083    ESTABLISHED 2624/./../node/bin/

3.测试

访问测试地址：http://192.168.43.99:5601/

[root@elk-master bin]# curl 127.0.0.1:9200/_cat/health?v

epoch      timestamp cluster    status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent

1598444891 12:28:11  es-cluster green          3        3    32  16    0    0        0            0                  -                100.0%

[root@elk-master bin]# curl http://192.168.43.99:9200/_cat/nodes?pretty

192.168.43.99  47 97 3 0.00 0.02 0.06 dilmrt * elk-master

192.168.43.100 29 96 1 0.00 0.01 0.05 dilmrt - elk-slave01

192.168.43.101 35 95 1 0.00 0.01 0.05 dilmrt - elk-slave02

[root@elk-master bin]#

---

四、FILEBEAT的安装配置

1.下载和安装Filebeat

[root@elk-master opt]# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.8.1-x86_64.rpm

[root@elk-master opt]# ll

total 668376

drwxr-xr-x 10 elsearch elsearch      167 Aug 26 11:05 elasticsearch-7.8.1

-rw-r--r--  1 root    root    318334518 Aug 26 10:39 elasticsearch-7.8.1-linux-x86_64.tar.gz

-rw-r--r--  1 root    root      28557354 Aug 26 14:39 filebeat-7.8.1-x86_64.rpm

drwxr-xr-x 13 elsearch elsearch      266 Aug 26 10:42 kibana-7.8.1-linux-x86_64

-rw-r--r--  1 root    root    337517217 Aug 26 10:39 kibana-7.8.1-linux-x86_64.tar.gz

[root@elk-master opt]#

[root@elk-master opt]# rpm -vi filebeat-7.8.1-x86_64.rpm

[root@elk-master opt]# rpm -qa | grep filebeat

filebeat-7.8.1-1.x86_64

2.编辑配置文件并启动

[root@elk-master opt]# grep -v '^#' /etc/filebeat/filebeat.yml | grep -v '^  #' | grep -v '^$'

filebeat.inputs:

- type: log

  enabled: false

  paths:

    - /var/log/*.log

    #- c:\programdata\elasticsearch\logs\*

filebeat.config.modules:

  path: ${path.config}/modules.d/*.yml

  reload.enabled: false

setup.template.settings:

  index.number_of_shards: 1

setup.kibana:

  host: "192.168.43.99:5601"

output.elasticsearch:

  hosts: ["192.168.43.99:9200"]

processors:

  - add_host_metadata: ~

  - add_cloud_metadata: ~

  - add_docker_metadata: ~

  - add_kubernetes_metadata: ~

[root@elk-master opt]# filebeat modules enable system

[root@elk-master opt]# filebeat setup

[root@elk-master opt]# service filebeat start

3.测试

使用kibana或者Elasticsearch-head进行数据的查看与分析。

1).创建1个索引：index_testelk

[root@elk-master opt]# curl -XPUT "http://192.168.43.99:9200/index_testelk"

{

"acknowledged": true,

"shards_acknowledged": true,

"index": "index_testelk"

}

2).获取这个索引信息：

[root@elk-master opt]# curl "http://192.168.43.99:9200/index_testelk"

{

"index_testelk": {

"aliases": {},

"mappings": {},

"settings": {

"index": {

"creation_date": "1598446212388",

"number_of_shards": "1",

"number_of_replicas": "1",

"uuid": "HA54WeTnTgSfmBwI3Gzkew",

"version": {

"created": "7080199"

},

"provided_name": "index_testelk"

}

}

}

}