JAVA定时器每日删除ES过期索引

JAVA定时器每日删除ES过期索引

背景：在Linux服务器上，设置一个后台进程，每天0点10分自动删除ELK中Elasticsearch中超过三个月的旧索引，使用Quartz实现。

废话少说，直接上代码！！！

目录：

1、源码：

2、调用方法：

1、源码：

（1）QuartzCli.java

package com.remoa.quartzCli;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.*;
import org.quartz.impl.StdSchedulerFactory;

/**
 * This class is used to use Quartz
 * @author remoa
 * @since 2017.09.18
 */
public class QuartzCli{
    //日志记录
    private static final Log LOG = LogFactory.getLog(QuartzCli.class);
    //后台流程每天0点10分开始跑
    //public static final String INDEX_DELETE_CRON = "0 10 0 * * ?";
    //后台流程每10秒开始跑
    private static final String INDEX_DELETE_CRON = "*/10 * * * * ?";
    //定义Job名字
    private static final String JOB_NAME = "Index_quartz_cli";

    /**
     * 启动定时任务
     */
    public void start(){
        System.out.println("Start quartz mode: " + INDEX_DELETE_CRON);
        LOG.info("Start quartz mode: " + INDEX_DELETE_CRON);
        //SchedulerFactory：提供用于获取调度程序实例的客户端可用句柄的机制
        SchedulerFactory schedulerFactory = new StdSchedulerFactory();
        //Scheduler：调度程序，接口，代表一个独立运行容器，调度程序维护JobDetail和触发器的注册表，一旦注册，调度程序负责执行作业。
        Scheduler scheduler = null;
        try {
            //设置调度程序的实现类
            scheduler = schedulerFactory.getScheduler();
        } catch (SchedulerException e) {
            LOG.error(e.getMessage(), e);
        }
        //实例化JobDetail调度任务，JobDetail存放作业的状态，JobDetail对象储存作业的侦听器、群组、数据映射、描述以及作业的其它属性。
        //newJob方法设置需要执行的Job任务类名
        JobBuilder jobBuilder = JobBuilder.newJob(MyJob.class);
        //withIdentity设置JobDetail的name和group
        jobBuilder.withIdentity(JOB_NAME, "remoaGroup");
        //JobDetail是一个接口，必须通过JobBuilder的build方法实例化JobDetail
        JobDetail job = jobBuilder.build();
        //实例化Trigger触发器，实现对作业的调度。
        CronTrigger cronTrigger = (CronTrigger)TriggerBuilder.newTrigger()
                .withIdentity("Cron_Trigger" + System.currentTimeMillis(), "remoaGroup")
                .withSchedule(CronScheduleBuilder.cronSchedule(INDEX_DELETE_CRON))
                .startNow()
                .build();
        //JobDataMap是java map的具体实现，并添加了一些便利的方法用于存储与读取原生类型数据，可以为Job实例提供属性/配置
        job.getJobDataMap()
                .put("executor", this);
        try {
            //为调度程序设置job和trigger
            scheduler.scheduleJob(job, cronTrigger);
            //调用start开始调度
            scheduler.start();
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
        }
    }

    public static void main(String[] args) {
        QuartzCli quartzCli = new QuartzCli();
        quartzCli.start();
    }

}

（2）MyJob.java

package com.remoa.quartzCli;

import com.remoa.index.IndexProcess;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;

/**
 * This class is used to define job
 * @author remoa
 * @since 2017.09.18
 */
public class MyJob implements Job {
    private static final Log LOG = LogFactory.getLog(MyJob.class);

    //在Quartz中，所有的任务都必须实现Job接口，Job中只有一个execute(JobExecutionContext jobExecutionContext)实现方法，具体业务实现需要写在这个方法里，这个方法将会在触发器满足调度条件时触发。
    @Override
    public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
        System.out.println(sdf.format(new Date()) + "------------- start job");
        LOG.info(sdf.format(new Date()) + "------------- start job");
        InputStream input = MyJob.class.getClassLoader().getResourceAsStream("indexDelete.properties");
        Properties properties = new Properties();
        try {
            properties.load(input);
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
        }
        IndexProcess indexProcess = null;
        try {
            indexProcess = new IndexProcess(properties.getProperty("keystorePath"), properties.getProperty("truststorePath"), properties.getProperty("keypassword"), properties.getProperty("trustpassword"), properties.getProperty("elasticsearchHost"), properties.getProperty("elasticsearchClusterName"));
        } catch (Throwable throwable) {
            LOG.error(throwable.getMessage());
            throwable.printStackTrace();
        }
        System.out.println("get index-------------");
        List list = indexProcess.getIndex();
        System.out.println("Total number of lists :" + list.size());
        LOG.info("Total number of lists :" + list.size());
        String keyword = indexProcess.getKeyword();
        indexProcess.deleteOldIndex(list, keyword);
    }

}

（3）IndexProcess.java

package com.remoa.index;

import com.floragunn.searchguard.ssl.SearchGuardSSLPlugin;
import com.floragunn.searchguard.ssl.util.SSLConfigConstants;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;

/**
 * This class is used to delete old index more than 100 days in elasticsearch.
 * @author remoa
 * @since 2017.09.18
 */
public class IndexProcess {
    private TransportClient client;
    private AdminClient adminClient;
    public static final int DATE_NUMBER = -90;

    /**
     * 构造方法，提供连接Elasticsearch的相关参数并使用传输客户端TransportClient连接到Elasticsearch集群
     * @param keystorePath KeyStore签名证书位置
     * @param truststorePath Truststore签名证书位置
     * @param keypwd KeyStore签名证书密码
     * @param trustpwd Truststore签名证书密码
     * @param esHost Elasticsearch集群服务器
     * @param esClusterName Elasticsearch集群名称
     * @throws Throwable 异常不单独处理了，直接在方法中抛出
     */
    public IndexProcess(String keystorePath, String truststorePath, String keypwd, String trustpwd, String
            esHost, String esClusterName) throws Throwable{
        Settings settings = Settings.builder()
                //ES集群名称
                .put("cluster.name", esClusterName)
                .put("searchguard.ssl.transport.enabled", true)
                //私钥证书位置
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_FILEPATH, keystorePath)
                //公钥证书位置
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_FILEPATH, truststorePath)
                //keystore一般保存我们的私钥，用来加密解密或者为别人做签名。
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_PASSWORD, keypwd)
                //truststore里存放的是只包含公钥的数字证书。
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_PASSWORD, trustpwd)
                .put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, false)
                .put("path.home", ".")
                //使得客户端去嗅探整个集群的状态，把集群中其它机器的IP地址加到客户端中，设置为true则不用手动设置集群里所有机器的IP连接到客户端，它会自动帮助添加，并自动发现新加入集群的机器。
                .put("client.transport.sniff", true)
                .build();
        client = new PreBuiltTransportClient(settings, SearchGuardSSLPlugin.class)
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9301))
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9300));
        adminClient = client.admin();
    }

    /**
     * 根据索引名删除索引
     * @param index 索引名
     */
    public void deleteIndex(String index){
        adminClient.indices().
                delete(new DeleteIndexRequest(index)).
                actionGet();
    }

    /**
     * 得到全部索引
     * @return 返回List封装的全部索引的名字
     */
    public List getIndex(){
        List list = new ArrayList();
        String [] indices = adminClient.indices()
                .prepareGetIndex()
                .setFeatures()
                .get()
                .getIndices();
        for(int i = 0; i < indices.length; i++){
            list.add(indices[i]);
        }
        return list;
    }

    /**
     * 得到过滤的关键字，即xxxx.xx.xx(年月日字符串)，这里设置为90天
     * @return 过滤关键字
     */
    public String getKeyword(){
        Calendar calendar = Calendar.getInstance();
        calendar.add(Calendar.DATE, DATE_NUMBER);
        String year = String.valueOf(calendar.get(Calendar.YEAR));
        String month = String.valueOf(calendar.get(Calendar.MONTH) + 1);
        String day = String.valueOf(calendar.get(Calendar.DATE));
        if(month.length() == 1){
            month = "0" + month;
        }
        if(day.length() == 1){
            day = "0" + day;
        }
        String keyword = year + "." + month + "." + day;
        return keyword;
    }

    /**
     * 删除所有的超期的旧索引
     * @param list 索引列表
     * @param keyword 过滤关键字
     */
    public void deleteOldIndex(List list, String keyword){
        int count = 0;
        for(Iterator iter = list.iterator(); iter.hasNext(); ){
            String str = iter.next();
            //索引名称字段长度比10小的话就不作为删除对象了
            if(str.length() > 10){
                String dateStr = str.substring(str.length() - 10 ,str.length());//取出索引名称字段的结尾字段
                //索引名称字段最后不是以日期结尾的话也不作为删除对象
                String patternStr = "^[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$";
                if(dateStr.matches(patternStr)){
                    if(str.substring(str.length() - 10 ,str.length()).compareTo(keyword) < 0){
                        System.out.println("index： " + str + " is deleted");
                        //this.deleteIndex(str);
                        count++;
                    }
                }
            }
        }
        System.out.println("The total delete index is: " + count);
    }

}

（4）log4j2.xml

（5）pom.xml

    4.0.0
    com.remoa
    indexTest
    1.0-SNAPSHOT
    
        indexTest
        
        src/main/java 
        src/test/java
        
            
                src/main/resources
            
        
        
            
                src/test/resources
            
        
        
            
            
                maven-compiler-plugin
                3.3
                
                    
                    1.8
                    
                    1.8
                
            
            
                
                
                org.apache.maven.plugins
                maven-shade-plugin
                2.4.3
                
                    
                        package
                        
                            shade
                        
                        
                            
                                
                                    reference.conf
                                
                            
                            
                                
                                    *:*
                                    
                                        META-INF/*.SF
                                        META-INF/*.DSA
                                        META-INF/*.RSA
                                    
                                
                            
                            true
                            jar-with-dependencies
                        
                    
                
            
        
    
    
        
        
            org.quartz-scheduler
            quartz
            2.2.1
        
        
            com.floragunn
            search-guard-5
            5.2.2-15
        
        
        
            com.floragunn
            search-guard-ssl
            5.2.2-22
        
        
        
            org.elasticsearch.client
            transport
            5.2.2
        
        
            org.apache.logging.log4j
            log4j-api
            2.7
        
        
            org.apache.logging.log4j
            log4j-core
            2.7
        
        
        
            org.slf4j
            slf4j-log4j12
            1.6.6
        
    

2、调用方法：

使用mvn进行打包部署后，进入target目录，输入nohup java -cp /root/remoa/indexTest/target/indexTest-1.0-SNAPSHOT-jar-with-dependencies.jar com.remoa.quartzCli.QuartzCli &将其设置为后台进程即可。