Elasticsearch 6.5 实现冷热数据分离

一、安装Elasticsearch集群web管理工具Cerebro

1.1 下载

https://github.com/lmenezes/cerebro/releases

[root@es-master1 software]# wget https://github.com/lmenezes/cerebro/releases/download/v0.8.1/cerebro-0.8.1.zip

1.2 安装及配置

[root@es-master1 software]# unzip cerebro-0.8.1.zip
[root@es-master1 software]# ln -s cerebro-0.8.1 cerebro
[root@es-master1 software]# cd cerebro
[root@es-master1 cerebro]# ll
total 16
drwxr-xr-x 2 root root   40 Dec  5 10:40 bin
drwxr-xr-x 3 root root  103 Dec  5 10:40 conf
drwxr-xr-x 2 root root 8192 Dec  5 10:40 lib
-rw-r--r-- 1 root root 1081 Jun 20 11:03 README.md
[root@es-master1 cerebro]# cd conf
[root@es-master1 conf]# vim application.conf
hosts = [
  {
    host = "http://es-master1.linuxplus.com:9200"
    name = "linuxplus"
    auth = {
         username = "root"
         password = "123456"
     }
  }
]
[root@es-master1 conf]# cd ..
[root@es-master1 cerebro]# nohup ./bin/cerebro -Dhttp.port=1234 -Dhttp.address=IP &
[1] 77172

二、配置Elasticsearch实现冷热数据分离

节点名称	服务器类型	存储数据
es-master1	SATA	元数据
es-master2
es-master3
es-hot1	SSD	Hot
es-hot2
es-hot3
es-stale1	SATA	Cold
es-stale2

2.1 配置Master节点

• Master1节点配置（其他节点配置类似）

[root@es-master1 ~]# cd /etc/elasticsearch/
[root@es-master1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-master1.linuxplus.com
node.attr.rack: r6
node.master: true
node.data: false
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300","es-hot1.linuxplus.com:9300","es-hot2.linuxplus.com:9300","es-hot3.linuxplus.com:9300","es-stale1.linuxplus.com:9300","es-stale2.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false

2.2 配置Hot节点

• Hot1节点配置（其他节点配置类似）

[root@es-hot1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-hot1.linuxplus.com     #需要修改
node.attr.rack: r1
node.master: false
node.data: true
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.10.10.24           #需要修改
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false
node.attr.box_type: hot
[root@es-hot1 elasticsearch]# /etc/init.d/elasticsearch start

在Hot节点打tag

node.attr.box_type: hot

2.3 配置Stale节点

• Stale1节点配置（其他节点配置类似）

[root@es-stale1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-stale1.linuxplus.com
node.attr.rack: r1
node.master: false
node.data: true
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.10.10.27
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false
node.attr.box_type: cold
[root@es-stale1 elasticsearch]# /etc/init.d/elasticsearch start

在Stale节点打tag

node.attr.box_type: cold

三、实现热数据写入指定节点

通过Cerebro修改模板添加如下配置

"settings": {
    "index": {
      "number_of_shards": "3",
      "auto_expand_replicas": "0-1",
      "routing": {
        "allocation": {
          "require": {
            "box_type": "hot"
          }
        }
      }
    }
  },

通过shell脚本将Hot数据（保留7天）迁移到Stale

#!/bin/bash
Time=$(date -d "1 week ago" +"%Y.%m.%d")
Hostname=$(hostname)
arr=("cron" "messages" "secure" "tomcat" "nginx-access" "nginxtcp" "nginxerror" "windows" ".monitoring-es-6" ".monitoring-beats-6" ".monitoring-kibana-6" ".monitoring-logstash-6" "metricbeat-6.5.3")
for var in ${arr[@]}
do
    curl -H "Content-Type: application/json" -XPUT http://$Hostname:9200/$var-$Time/_settings?pretty -d'
    { 
       "settings": { 
             "index.routing.allocation.require.box_type": "cold"
        } 
    }'
done