elk环境搭建

一、说明

这篇文章是2020年6月份写在自己博客的，组件版本应该都比较低了。

二、组件

filebeat
kafka
zookeeper
elasticsearch
kibana

三、参考链接：

# 1 json在线解析
https://www.json.cn/
# 2 grok debug
http://grokdebug.herokuapp.com/
http://grokdebug.herokuapp.com/patterns
# 3 template接口
https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html
# 4 elk官网指南
https://www.elastic.co/guide/index.html
https://www.elastic.co/

四、服务器

172.16.1.30 kc-log-01                            日志所在服务器
172.16.1.94 kc-online-log-platform-001   elasticsearch  kafka  zookeeper
172.16.1.93 kc-online-log-platform-002  elasticsearch  kafka  zookeeper
172.16.1.92 kc-online-log-platform-003  elasticsearch  kafka  zookeeper  kibana  logstash

五、JAVA JDK安装

# 1 获取java jdk安装包
[root@kc-online-log-platform-001 ~]# ls /opt/
elk  jdk1.8.0_211  kafka
# 2 配置环境变量
[root@kc-online-log-platform-001 ~]# cat /etc/profile.d/java.sh
export JAVA_HOME=/opt/jdk1.8.0_211
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$PATH:$JAVA_HOME/bin
# 3 应用环境变量
[root@kc-online-log-platform-001 ~]# source /etc/profile.d/java.sh

六、目录结构

[root@kc-online-log-platform-001 elk]# ls /data/
elk  kafka  zk
[root@kc-online-log-platform-001 elk]# ls /data/elk/elasticsearch/
config  data
[root@kc-online-log-platform-001 elk]# ls /data/zk/
zkdatalog  zookeeper

七、elasticsearch集群搭建

# kc-online-log-platform-001
[root@kc-online-log-platform-001 elk]# cat docker-compose.yaml 
version: '3'
services:
  kclog-online-node03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
    container_name: kclog-online-node03
    restart: always
    environment:
      - node.name=es-node-03
      - network.publish_host=172.16.1.94
      - network.host=0.0.0.0
      - discovery.seed_hosts=172.16.1.92,172.16.1.93,172.16.1.94
      - cluster.name=kc-log-center
      - cluster.initial_master_nodes=172.16.1.92,172.16.1.93,172.16.1.94
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /data/elk/elasticsearch/data:/usr/share/elasticsearch/data
      - /data/elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300

# kc-online-log-platform-002
[root@kc-online-log-platform-002 ~]# cat /opt/elk/docker-compose.yaml 
version: '3'
services:
  kclog-online-node02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
    container_name: kclog-online-node02
    restart: always
    environment:
      - node.name=es-node-02
      - network.publish_host=172.16.1.93
      - network.host=0.0.0.0
      - discovery.seed_hosts=172.16.1.92,172.16.1.93,172.16.1.94
      - cluster.name=kc-log-center
      - cluster.initial_master_nodes=172.16.1.92,172.16.1.93,172.16.1.94
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /data/elk/elasticsearch/data:/usr/share/elasticsearch/data
      - /data/elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300

# kc-online-log-platform-003
[root@kc-online-log-platform-003 ~]# cat /opt/elk/docker-compose.yaml 
version: '3'
services:
  kclog-online-node01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
    container_name: kclog-online-node01
    restart: always
    environment:
      - node.name=es-node-01
      - network.publish_host=172.16.1.92
      - network.host=0.0.0.0
      - discovery.seed_hosts=172.16.1.92,172.16.1.93,172.16.1.94
      - cluster.name=kc-log-center
      - cluster.initial_master_nodes=172.16.1.92,172.16.1.93,172.16.1.94
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /data/elk/elasticsearch/data:/usr/share/elasticsearch/data
      - /data/elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300

# 上述三台服务器执行一下操作：
cd /opt/elk/
docker-compose up -d

八、常用api

# elasticsearch api
# 1 获取索引
curl 'localhost:9200/_cat/indices?v'
curl -ulogstash:jflogstash123 '172.18.238.3:9200/_cat/indices?'   带密码查询
# 2 创建索引
curl -XPUT 'localhost:9200/customer’
# 3 插入数据
curl -XPUT 'localhost:9200/customer/external/1'-d ' { "name": "John Doe”}'
# 4 查询数据
curl 'localhost:9200/customer/external/1'
# 5 删除索引
curl -XDELETE 'localhost:9200/customer'
# 6 查看已安装的插件
curl http://127.0.0.1:9200/_cat/plugins
# 7 查看索引信息
curl -XGET "http://localhost:9200/kc-online-log-2020.06.21/_mapping"
# 8 创建template 其中：kc_online_log_template为创建的template
curl -XPUT -H "Content-Type: application/json" "http://localhost:9200/_template/kc_online_log_template" -d ''
# 9 集群状态查看
curl 'localhost:9200/_cat/health?v
# 10 查看集群node
curl 'localhost:9200/_cat/nodes?v'

九、kafka+zookeeper集群

9.1、kc-online-log-platform-001

9.1.1、zookeeper

[root@kc-online-log-platform-001 config]# grep -v "^#\|^$" /opt/kafka/config/zookeeper.properties
tickTime=2000
initLimit=5
syncLimit=2
dataDir=/data/zk/zookeeper/
dataLogDir=/data/zk/zkdatalog/
clientPort=2181
maxClientCnxns=0
server.1=172.16.1.94:2888:3888
server.2=172.16.1.93:2888:3888
server.3=172.16.1.92:2888:3888
[root@kc-online-log-platform-001 config]# cat /data/zk/zookeeper/myid
1

9.1.2、kafka

[root@kc-online-log-platform-001 config]# cat /opt/kafka/config/server.properties
broker.id=0
listeners=PLAINTEXT://172.16.1.94:9092
host.name=172.16.1.94
port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/kafka-logs/
num.partitions=3
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.16.1.94:2181,172.16.1.93:2181,172.16.1.92:2181
zookeeper.connection.timeout.ms=6000

9.2、kc-online-log-platform-001

9.2.1、zookeeper

[root@kc-online-log-platform-002 ~]# grep -v "^#\|^$" /opt/kafka/config/zookeeper.properties
tickTime=2000
initLimit=5
syncLimit=2
dataDir=/data/zk/zookeeper/
dataLogDir=/data/zk/zkdatalog/
clientPort=2181
maxClientCnxns=0
server.1=172.16.1.94:2888:3888
server.2=172.16.1.93:2888:3888
server.3=172.16.1.92:2888:3888
[root@kc-online-log-platform-002 ~]# cat /data/zk/zookeeper/myid
2

9.2.2、kafka

[root@kc-online-log-platform-002 ~]# cat /opt/kafka/config/server.properties
broker.id=1
listeners=PLAINTEXT://172.16.1.93:9092
host.name=172.16.1.93
port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/kafka-logs/
num.partitions=3
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.16.1.94:2181,172.16.1.93:2181,172.16.1.92:2181
zookeeper.connection.timeout.ms=6000

9.3、kc-online-log-platform-001

9.3.1、zookeeper

[root@kc-online-log-platform-003 kafka]# grep -v "^#\|^$" /opt/kafka/config/zookeeper.properties
tickTime=2000
initLimit=5
syncLimit=2
dataDir=/data/zk/zookeeper/
dataLogDir=/data/zk/zkdatalog/
clientPort=2181
maxClientCnxns=0
server.1=172.16.1.94:2888:3888
server.2=172.16.1.93:2888:3888
server.3=172.16.1.92:2888:3888
[root@kc-online-log-platform-003 kafka]# cat /data/zk/zookeeper/myid
3

9.3.2、kafka

[root@kc-online-log-platform-003 kafka]# cat /opt/kafka/config/server.properties
broker.id=2
listeners=PLAINTEXT://172.16.1.92:9092
host.name=172.16.1.92
port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/kafka-logs/
num.partitions=3
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.16.1.94:2181,172.16.1.93:2181,172.16.1.92:2181
zookeeper.connection.timeout.ms=6000

十、启动

# 三台服务器执行如下命令：
nohup /opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties &
nohup /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties &

十一、测试集群有效性

# 创建一个topic：fzh
kafka-topics.sh --create --zookeeper inte-zk-node-1:2181 --replication-factor 3 --partitions 1 --topic fzh
# 其它节点查看该topic是否存在：
kafka-topics.sh --describe --zookeeper kc-online-log-platform-001:2181 --topic fzh
kafka-topics.sh --describe --zookeeper kc-online-log-platform-002:2181 --topic fzh
kafka-topics.sh --describe --zookeeper kc-online-log-platform-003:2181 --topic fzh
创建一个topic，发现都能查到该topic，证明：集群成功。
# 查看topic信息：
./kafka-console-consumer.sh --bootstrap-server 172.16.1.94:9092 --topic kc-online-log --from-beginning

十二、filebeat

12.1、filebeat配置

[root@online-k8s2-master filebeat]# cat filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /mnt/online-data-4/thirdhub*/logs/app.log
    - /mnt/online-data-4/obc-stock-log/logs/app.log
  multiline:
    pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
    negate: true
    match: after
output.kafka:
  hosts: ["172.16.1.94:9092"]
  topic: "kc-online-log"

12.2、filebeat启动

./filebeat -c filebeat.yml

十三、logstash

13.1、logstash配置

[root@kc-online-log-platform-003 kafka]# cat /opt/logstash/config/logstash.conf 
input {
  kafka {
    bootstrap_servers => "172.16.1.93:9092"
    topics => ["kc-online-log"]
    codec => json
  }
}
filter {
    grok {
      match => { "message" => "%{USERNAME:day} %{NOTSPACE:time} %{NOTSPACE:threadid} %{LOGLEVEL:level} %{GREEDYDATA:msg}" }
    }
    grok {
      match => { "[log][file][path]" => ".*/%{USERNAME:app_name}/logs" }
    }
}
output {
  elasticsearch {
    hosts => "http://172.16.1.92:9200"
    index => "kc-online-log-%{+YYYY.MM.dd}"
  }
}

13.2、启动

[root@kc-online-log-platform-003 logstash]# ./bin/logstash -f config/logstash.conf

十四、kibana

[root@kc-online-log-platform-003 kibana]# cat docker-compose.yaml
version: '3'
services:
  kclog-kibana:
    image: docker.elastic.co/kibana/kibana:7.8.0
    container_name: kclog-kibana
    restart: always
    environment:
      elasticsearch.hosts: "http://172.16.1.93:9200"
      server.host: "0.0.0.0"
    volumes:
      - /data/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - "5601:5601"
[root@kc-online-log-platform-003 kibana]# cat /data/elk/kibana/config/kibana.yml
server.name: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://172.16.1.92:9200" ]

十五、elasticsearch索引配置

在kibana上进行词语搜索时，经常会发现搜索的内容不准确，eg：搜索thread:"app-app1-app2"的时候，有app或app1或app2的thread都会被检索到，而我们实际想要的是只有"app-app1-app2"的thread信息。
查看索引mapping信息

curl -XGET "http://localhost:9200/kc-online-log-2020.06.21/_mapping"

用在线json分析工具分析10.1得到的结果：

curl -XPUT -H "Content-Type: application/json" "http://localhost:9200/_template/kc_online_log_template" -d '
{
    "index_patterns": ["kc-online-log-*"],
    "mappings": {
        "properties": {
            "app_name": {
                "type": "keyword"
            },
            "agent": {
                "properties": {
                    "hostname": {
                        "type": "keyword"    # 这个地方是解决问题的关键，至于template的结构书写，可以直接复制黏贴10.1得到的结果，根据实际情况来。
                    }
                }
            }
        }
    }
}'