EFK之elasticSearch自动清理方案

 1 简介

本方案实现的功能点：输入日志文件、按照指定的日期（天数）清理数据、按照指定的磁盘路径使用率阈值清理数据、清除指定的索引数据，并配合crond实现定时监控和清理索引数据，以确保磁盘空间健康。

2 使用方法

脚本名称为：clearIndex.sh

2.1 重要变量解释

DISK_PATH='/home'

解释：监控的磁盘路径，此处监控/home路径的使用率

DISK_THRESHOLD=70

解释：监控的磁盘路径的使用率百分比，此处是监控的磁盘路径使用率超过70%就符合启动清理

TARGET_DATE_NUMBER=180

解释：清理指定天数之前对应日期的数据，此处是清理的是180天前对应日期那天的所有数据

2.1 检查磁盘路径使用率并清理数据

./clearIndex.sh start_for_diskThreshold

2.2 清理指定天数前日期的数据

./clearIndex.sh start_for_specifiedDate

2.3 清理指定索引数据

./clearIndex.sh start_for_index test.helloword-20220321

2.4 自动监控并清理数据

配合crond任务管理器，定时执行2.1，2.2即可实现自动监控和自动清理，确保磁盘空间健康。

#!/bin/bash
#Usage: Automatic clearing the indexes of ES!
#Maintenance: xxxxx
#Description: Here are offered two method of deletion indexes.
#Method 1: Specify the date that you expect to clear the indexes.
#Method 1: Specify the server disk space usage threshold for clearing indexes.
#

CURDIR="$( cd "$( dirname "$0"  )" && pwd  )"

#Log file path
LOGPATH=$CURDIR/logs

if [[ ! -d $LOGPATH ]];then
	mkdir $LOGPATH	
fi

#################################################################################################################
#
#Configuration:
#The user/password of es
ES_USER='xxxxxx'
ES_PASS='xxxxxx'

#The disk space path
DISK_PATH='/home'
DISK_THRESHOLD=70
CHECK_DISK=`df -h | grep $DISK_PATH | tr '%' ' ' | awk '$5>'$DISK_THRESHOLD'{print $5}' | wc -l`

#The url of es
ES_URL='https://localhost:9200/'
#Specify the prefix of index needing to delete.
PREFIX=test

#Specify the date of deletion indexes by many different date formats
#exp.: 1 month ago   ；  +%Y-%m-%d
TARGET_DATE_NUMBER=180
#
#################################################################################################################

#TARGET_DATE=`date -d "3 week ago" +%Y%m%d`
#TARGET_DATE=`date -d "6 month ago" +%Y%m%d`
TARGET_DATE=`date -d "$TARGET_DATE_NUMBER day ago" +%Y%m%d`

#The current date
#CURRENT_DATE=`date +%Y%m%d--%T`
CURRENT_DATE=`date +%Y%m%d` 

#Query the all indexes
#curl -k --basic -u ${ES_USER}:${ES_PASS} ${ES_URL}_cat/indices?v

#Print the log information and output it into a file.
display(){
    local type=$1
    local msg=${@:2}
	curDateTime=`date +'%Y-%m-%d %H:%M:%S.%N'`
    if [[ $type = "[Info]" ]]; then
        echo -e "\\033[1;36;40m[Info] ${curDateTime} $msg \\033[0m"
    elif [[ $type = "[Error]" ]]; then
        echo -e "\\033[1;31;40m[Error] ${curDateTime} $msg \\033[0m"
    elif [[ $type = "[Exec]" ]]; then
        echo -e "\\033[1;33;40m[Exec] ${curDateTime} $msg \\033[0m"
    elif [[ $type = "[Success]" ]]; then
        echo -e "\\033[1;32;40m[Success] ${curDateTime} $msg \\033[0m"
    else
        echo -e "${curDateTime} $@"
    fi
	echo "${curDateTime} $type $msg " >> ${LOGPATH}/${USER}-indexClear-${dateNYR}.log
}

#To countdown 
function countdownTimer(){
	#countdown
	time=$1
	for((i=time;i>0;i--))
	do
		display [Exec] "${i} seconds to start："
		sleep 1
	done
	
}

#Clear the specified index
function clear_index(){
	specifiedIndex=$1
	if [[ ! -n "$specifiedIndex" ]];then
		display [Error] "The specified index to be deleted mustn't be null!"
		return
	fi
	
	curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}${specifiedIndex}
	if [ $? -eq 0 ];then
		display [Success] "-->${CURRENT_DATE} Clear '${specifiedIndex}' success.."
	else
		display [Error] "-->${CURRENT_DATE} Clear '${specifiedIndex}' failed.."
	fi
	

}

#Clear the indexes by specified date
function clear_indexes(){
	specifiedPrefix=$1
	specifiedDate=$2
	display [Info] "Query the target indexes '${specifiedPrefix}.*${specifiedDate}' ready to clear!"
	#Query the specified indexes by the date format
	#curl -k --basic -u ${ES_USER}:${ES_PASS} -XGET ${ES_URL}{$PREFIX}.*${TARGET_DATE}
	TARGET_INDEXES=`curl -k --basic -u ${ES_USER}:${ES_PASS} -XGET ${ES_URL}_cat/indices?v | awk '{print $3}' | grep "${specifiedPrefix}.*${specifiedDate}"`

	for index in ${TARGET_INDEXES};do
		#Delete the indexes specifying date format
		#curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}{$PREFIX}.*${TARGET_DATE}
		curl -k --basic -u ${ES_USER}:${ES_PASS} -XDELETE ${ES_URL}${index}
		if [ $? -eq 0 ];then
			display [Success] "-->${CURRENT_DATE} Clear '${index}' success.."
		else
			display [Error] "-->${CURRENT_DATE} Clear '${index}' failed.."
		fi
	done 

}


#Start to clear the indexes by the disk threshold.
function start_for_diskThreshold(){
	if [ $CHECK_DISK -gt 0 ];then
		#checked overload
		currentDiskUsage=`df -h | grep $DISK_PATH | awk '{print $5}'`
		display [Exec] "Warning! The current disk usage is $currentDiskUsage (>=${DISK_THRESHOLD}% ) , the disk is overload! It will be clearing the indexes!"
		clear_indexes $PREFIX $TARGET_DATE
		display [Info] "This round to clear the indexes is completed,continuing to go next check round!"
		countdownTimer 10
		#Modify the 'TARGET_DATE' to add one day backward.
		TARGET_DATE_NUMBER=$((TARGET_DATE_NUMBER - 1))
		TARGET_DATE=`date -d "$TARGET_DATE_NUMBER day ago" +%Y%m%d`
		display [Exec] "TARGET_DATE_NUMBER is $TARGET_DATE_NUMBER!  TARGET_DATE is $TARGET_DATE"
		main
	else
		#checked normal
		display [Success] "The disk is nomal! Everything is ok, today is a good day! The round to check is completed!"
	fi

}

#Start to clear the indexes by the prefix and the date 
function start_for_specifiedDate(){
	clear_indexes $PREFIX $TARGET_DATE
}

#Start to clear the index by the index.
function start_for_index(){
	clear_index $1
}

case $1 in  
	byDate)
		start_for_specifiedDate
	;;
	
	byDisk)
		start_for_diskThreshold
	;;	
	byIndex)
		start_for_index $2
	;;	
	
   *)  
      display [Info]  "Usage: {byDate|byDisk|byIndex index}"  
   ;;  
esac