1. es 快照备份和还原 原理
es 快照.png
版本兼容
官网说明https://www.elastic.co/guide/en/elasticsearch/reference/7.17/snapshot-restore.html
ES8 版本兼容.png
es 快照兼容.png
2. 下载查插件的地址
https://artifacts.elastic.co/downloads/elasticsearch-plugins/repository-hdfs/repository-hdfs-7.12.0.zip
3. 下载后放入到elasticsearch 安装目录中的elasticsearch/plugins 中
[root@basenode repository-hdfs]# pwd
/root/tools/elasticsearch/plugins/repository-hdfs
[root@basenode repository-hdfs]# ll
总用量 18628
-rw-r--r-- 1 root root 41123 7月 3 2019 commons-cli-1.2.jar
-rw-r--r-- 1 root root 335042 7月 3 2019 commons-codec-1.11.jar
-rw-r--r-- 1 root root 588337 7月 3 2019 commons-collections-3.2.2.jar
-rw-r--r-- 1 root root 298829 7月 3 2019 commons-configuration-1.6.jar
-rw-r--r-- 1 root root 185140 7月 3 2019 commons-io-2.4.jar
-rw-r--r-- 1 root root 284220 7月 3 2019 commons-lang-2.6.jar
-rw-r--r-- 1 root root 62050 7月 3 2019 commons-logging-1.1.3.jar
-rw-r--r-- 1 root root 1648200 7月 3 2019 guava-11.0.2.jar
-rw-r--r-- 1 root root 45930 3月 26 2020 hadoop-annotations-2.8.5.jar
-rw-r--r-- 1 root root 123295 3月 26 2020 hadoop-auth-2.8.5.jar
-rw-r--r-- 1 root root 29917 3月 26 2020 hadoop-client-2.8.5.jar
-rw-rw-r-- 1 root root 3948772 3月 18 2021 hadoop-common-7.12.0.jar
-rw-r--r-- 1 root root 4857023 3月 26 2020 hadoop-hdfs-2.8.5.jar
-rw-r--r-- 1 root root 4093994 3月 26 2020 hadoop-hdfs-client-2.8.5.jar
-rw-r--r-- 1 root root 1485102 7月 3 2019 htrace-core4-4.0.1-incubating.jar
-rw-r--r-- 1 root root 34447 3月 18 2021 LICENSE.txt
-rw-r--r-- 1 root root 63731 7月 3 2019 log4j-1.2-api-2.11.1.jar
-rw-r--r-- 1 root root 23241 7月 3 2019 log4j-slf4j-impl-2.11.1.jar
-rw-rw-r-- 1 root root 165628 3月 18 2021 NOTICE.txt
-rw-rw-r-- 1 root root 1748 3月 18 2021 plugin-descriptor.properties
-rw-r--r-- 1 root root 3739 3月 18 2021 plugin-security.policy
-rw-r--r-- 1 root root 533455 7月 3 2019 protobuf-java-2.5.0.jar
-rw-rw-r-- 1 root root 34685 3月 18 2021 repository-hdfs-7.12.0.jar
-rw-r--r-- 1 root root 105112 7月 3 2019 servlet-api-2.5.jar
-rw-r--r-- 1 root root 25689 7月 3 2019 slf4j-api-1.6.2.jar
[root@basenode repository-hdfs]#
3. 重启Elasticsearch 并且验证
docker restart 949389b28b4e
3.1验证：
GET /_cat/plugins?v
http://192.168.1.180:9200/_cat/plugins?v
es插件验证.png
4. 创建仓库
hdfs 创建仓库文件夹 hdfs dfs -mkdir -p /elasticsearch/backups/my_hdfs_repository
[root@node01 ~]# hdfs dfs -mkdir -p /elasticsearch/backups/my_hdfs_repository
[root@node01 ~]# hdfs dfs -ls /
Found 28 items
drwxr-xr-x - Administrator hdfs 0 2021-12-27 00:50 /FlinkFileSink
drwxrwxrwt - yarn hadoop 0 2020-08-07 03:02 /app-logs
drwxr-xr-x - hdfs hdfs 0 2020-08-07 00:40 /apps
drwxr-xr-x - yarn hadoop 0 2020-08-06 23:58 /ats
drwxr-xr-x - hdfs hdfs 0 2020-08-07 00:04 /atsv2
drwxr-xr-x - root hdfs 0 2021-10-02 18:25 /cdc-test
drwxr-xr-x - hdfs hdfs 0 2021-10-02 18:57 /cdc2.0-test
drwxr-xr-x - root hdfs 0 2022-07-30 21:34 /elasticsearch
drwxr-xr-x - Administrator hdfs 0 2020-11-12 00:20 /fink-checkpoint
drwxr-xr-x - root hdfs 0 2022-02-09 23:08 /flink
drwxr-xr-x - Administrator hdfs 0 2021-12-27 00:50 /flink-checkpoint
drwxr-xr-x - root hdfs 0 2022-02-13 20:53 /flink-hudi
drwxr-xr-x - hdfs hdfs 0 2020-08-07 00:21 /warehouse
[root@node01 ~]#
命令：
PUT _snapshot/my_hdfs_repository
{
"type": "hdfs",
"settings": {
"uri": "hdfs://192.168.1.161:8020",
"path": "/elasticsearch/backups/my_hdfs_repository",
"conf.dfs.client.read.shortcircuit": "false"
}
}
--------------------------------------------------------
{
"acknowledged" : true
}
4.1 查看仓库快照
# 查看仓库快照
GET /_cat/repositories
---------------------------------
my_hdfs_repository hdfs
4.2 指定索引创建快照
PUT /_snapshot/my_hdfs_repository/my_hdfs_repository_snapshot100
{
"indices": "wudbes",
"ignore_unavailable": false,
"include_global_state": true,
"partial": false
}
--------------------------------------------------
{
"accepted" : true
}
4.3 全部索引快照
PUT /_snapshot/my_hdfs_repository/my_hdfs_repository_snapshot101
{
}
--------------------------------------------------
{
"accepted" : true
}
4.4 查看快照进度
GET /_cat/snapshots/my_hdfs_repository?v&s=id
-------------------------------------------------
id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
my_hdfs_repository_snapshot100 SUCCESS 1659193380 15:03:00 1659193384 15:03:04 3.2s 5 5 0 5
my_hdfs_repository_snapshot101 SUCCESS 1659193454 15:04:14 1659193456 15:04:16 2.4s 9 9 0 9
4.5 查看快照状态
GET /_snapshot/my_hdfs_repository/my_hdfs_repository_snapshot100/_status
-------------------------------------------------
{
"snapshots" : [
{
"snapshot" : "my_hdfs_repository_snapshot100",
"repository" : "my_hdfs_repository",
"uuid" : "iaZaOgATR9GNsYZYhCmlsw",
"state" : "SUCCESS",
"include_global_state" : true,
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 5,
"failed" : 0,
"total" : 5
},
"stats" : {
"incremental" : {
"file_count" : 107,
"size_in_bytes" : 2342202
},
"total" : {
"file_count" : 107,
"size_in_bytes" : 2342202
},
"start_time_in_millis" : 1659193380831,
"time_in_millis" : 3207
},
"indices" : {
".kibana_task_manager_7.12.0_001" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 1,
"failed" : 0,
"total" : 1
},
"stats" : {
"incremental" : {
"file_count" : 71,
"size_in_bytes" : 111765
},
"total" : {
"file_count" : 71,
"size_in_bytes" : 111765
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 2806
},
"shards" : {
"0" : {
"stage" : "DONE",
"stats" : {
"incremental" : {
"file_count" : 71,
"size_in_bytes" : 111765
},
"total" : {
"file_count" : 71,
"size_in_bytes" : 111765
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 2806
}
}
}
},
".apm-custom-link" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 1,
"failed" : 0,
"total" : 1
},
"stats" : {
"incremental" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"total" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"start_time_in_millis" : 1659193381433,
"time_in_millis" : 0
},
"shards" : {
"0" : {
"stage" : "DONE",
"stats" : {
"incremental" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"total" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"start_time_in_millis" : 1659193381433,
"time_in_millis" : 0
}
}
}
},
".apm-agent-configuration" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 1,
"failed" : 0,
"total" : 1
},
"stats" : {
"incremental" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"total" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"start_time_in_millis" : 1659193380831,
"time_in_millis" : 401
},
"shards" : {
"0" : {
"stage" : "DONE",
"stats" : {
"incremental" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"total" : {
"file_count" : 1,
"size_in_bytes" : 208
},
"start_time_in_millis" : 1659193380831,
"time_in_millis" : 401
}
}
}
},
"wudbes" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 1,
"failed" : 0,
"total" : 1
},
"stats" : {
"incremental" : {
"file_count" : 10,
"size_in_bytes" : 12709
},
"total" : {
"file_count" : 10,
"size_in_bytes" : 12709
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 1403
},
"shards" : {
"0" : {
"stage" : "DONE",
"stats" : {
"incremental" : {
"file_count" : 10,
"size_in_bytes" : 12709
},
"total" : {
"file_count" : 10,
"size_in_bytes" : 12709
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 1403
}
}
}
},
".kibana_7.12.0_001" : {
"shards_stats" : {
"initializing" : 0,
"started" : 0,
"finalizing" : 0,
"done" : 1,
"failed" : 0,
"total" : 1
},
"stats" : {
"incremental" : {
"file_count" : 24,
"size_in_bytes" : 2217312
},
"total" : {
"file_count" : 24,
"size_in_bytes" : 2217312
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 2806
},
"shards" : {
"0" : {
"stage" : "DONE",
"stats" : {
"incremental" : {
"file_count" : 24,
"size_in_bytes" : 2217312
},
"total" : {
"file_count" : 24,
"size_in_bytes" : 2217312
},
"start_time_in_millis" : 1659193381232,
"time_in_millis" : 2806
}
}
}
}
}
}
]
}
4.6 es 备份hdfs 的目录文件
es-备份hdfs.png
4.7 删除快照
DELETE /_snapshot/my_hdfs_repository/smy_hdfs_repository_snapshot100
--------------------------------------------------
{
"accepted" : true
}
5. 快照还原
要先将快照删除之前的索引在还原，如果是全部的索引 还原 就将全部的索引删除，但是.kibana的索引无法删除要用poostmain 删除 就不会建了 因为用kibana 会自动创建索引。
POST /_snapshot/my_hdfs_repository/my_hdfs_repository_snapshot100/_restore
{
"indices": "wudbes",
"index_settings": {
"index.number_of_replicas": 0
},
"ignore_index_settings": [
"index.refresh_interval"
]
}
------------------------------------------
{
"accepted" : true
}
查看
GET /_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open .kibana_task_manager_7.12.0_001 fj0u3Mi9Th-i4bjgiA3JQw 1 0 9 335 165.8kb 165.8kb
green open .apm-custom-link Q-Aexfq3TV2rt7G_YdALyQ 1 0 0 0 208b 208b
green open .apm-agent-configuration z19tPJXYRKeppcMT_r3xnw 1 0 0 0 208b 208b
green open wudbes SlWWdlztRci3BPY6HGSVwg 1 0 3 0 12.4kb 12.4kb
green open .kibana_7.12.0_001 fFsDUjaNSjuWq1-13tQMGw 1 0 60 36 4.2mb 4.2mb
green open .kibana-event-log-7.12.0-000001 OsA26AZESX6wvi1Jvy1xTQ 1 0 1 0 5.6kb 5.6kb
yellow open wudl_book qfo8coDoTYuTzLitaNEAUA 1 1 3 0 14.2kb 14.2kb
yellow open wubigdata rgLSb-jESwWMebzacl0ESw 1 1 4 1 10.8kb 10.8kb
已经还原成功
POST /_snapshot/repository1/snapshot-20211214-001/_restore
{
}
------------------------------------------
{
"accepted" : true
}