java读取elasticsearch全量数据并导出json文件

一：创建ES连接
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import java.io.IOException;
import java.net.InetAddress;

public class ElasticSearchUtil {

    private static TransportClient client;
    static {
        try {
            /*此处可穿件配置文件自行配置，代码中为方便直接写死
InputStream resourceAsStream = ElasticSearchUtil.class.getClassLoader().getResourceAsStream("elasticsearch.conf");*/
            String host = "10.0.221.73";
            int port = Integer.parseInt("9300");
            String clusterName = "my-application";
            Settings settings = Settings.builder()
                    .put("cluster.name", clusterName)
                    //.put("client.transport.sniff", true)
                    .build();
            client = new PreBuiltTransportClient(settings);
            //这里可以有多个，集群模式
            TransportAddress ta = new TransportAddress(
                    InetAddress.getByName(host), port);
            //addTransportAddresses(TransportAddress... transportAddress)，参数为一个可变参数
            client.addTransportAddresses(ta);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static TransportClient getTransportClient() {
        return client;
    }

    public static void close(TransportClient client) {
        if (client != null) {
            client.close();
        }
    }
}

 二： 创建文件输出流

import java.io.FileOutputStream;
import java.io.IOException;

public class JsonFormatTool {
    /**
     * 将字符串追加到文件已有内容后面
     *
     * @param fileFullPath 文件完整地址：/es.json
     * @param
    content 需要写入的
     */
    public static void writeFile(String fileFullPath,String content) {
        FileOutputStream fos = null;
        try {
            //true不覆盖已有内容
            fos = new FileOutputStream(fileFullPath, true);
            //写入
            fos.write(content.getBytes());
            // 写入一个换行
            fos.write("\r\n".getBytes());

        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            if(fos != null){
                try {
                    fos.flush();
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

三：读取ES数据并调用输出流输出json文件

import org.elasticsearch.action.search.*;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;

public class ScrollAll {
   public static void main(String[] args) {
        ScroolSearchID();
   }

    public static String ScroolSearchID() {
        //动态获取日期作为生成json文件的后缀
        Date date = new Date();
        DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
        String suffix = dateFormat.format(date);
        //获取ES连接
        Client client = ElasticSearchUtil.getTransportClient();

        //指定一个index和type
        SearchRequestBuilder search = client.prepareSearch("binloghash").setTypes("binloghash");
        //使用原生排序优化性能
        search.addSort("_doc", SortOrder.ASC);
        //设置每批读取的数据量,此处为测试环境为了测试批次设置值较小，生产可根据实际情况调参
        search.setSize(10);
        //默认是查询所有
        search.setQuery(QueryBuilders.queryStringQuery("*:*"));
        //设置 search context 维护1分钟的有效期
        search.setScroll(TimeValue.timeValueMinutes(1));

        //获得首次的查询结果
        SearchResponse scrollResp = search.get();
        //打印命中数量
        System.out.println("命中总数量：" + scrollResp.getHits().getTotalHits());
        //打印计数
        int count = 1;
        do {
            System.out.println("第" + count + "次打印数据：");
            //读取结果集数据
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                //System.out.println(hit.getSourceAsString());
                //获取ES数据
                String esData = hit.getSourceAsString();
                //按日期动态生成文件
                String fileFullPath = "/tmp/es"+suffix+".json";
                //调用输出流输出文件
                JsonFormatTool.writeFile(fileFullPath,esData);
            }
            count++;
            //将scorllId循环传递
            scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();

            //当searchHits的数组为空的时候结束循环，至此数据全部读取完毕
        } while (scrollResp.getHits().getHits().length != 0);
        return null;
    }
}

四：maven依赖

  4.0.0
  com.my.es
  EsToFile
  jar
  1.0-SNAPSHOT
  EsToFile Maven Webapp
  http://maven.apache.org
  
    UTF-8
  
  
    
      junit
      junit
      4.10
    
    
      org.elasticsearch.client
      transport
      6.4.2
    
    
      org.json
      json
      20180813
    
    
      org.elasticsearch
      elasticsearch-hadoop
      6.4.2
    
  

  
    
      
        org.apache.maven.plugins
        maven-compiler-plugin
        
          1.8
          1.8
        
      
      
        org.apache.maven.plugins
        maven-jar-plugin
        
          
            
              true
              false
              lib/
              cn.mymaven.test.TestMain
            
          
        
      
    
  

总结：需要注意的点1. es7以后 TransportClient被废弃，此代码不适用7版本以后，es7的java代码，只能使用restclient，建议采用 High-level-rest-client 的方式操作ES集群。

2. 以上java连接ES,注意在ES集群中的elasticsearch.yml添加开放9300端口配置