es的api使用

package com.bcht.bigdata.dsjmh.api.utils;


import com.bcht.bigdata.common.entity.ESIndexConfig;
import org.apache.http.HttpHost;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.*;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.*;
import org.elasticsearch.search.aggregations.bucket.filter.Filters;
import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator;
import org.elasticsearch.search.aggregations.bucket.histogram.*;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.util.*;

/**
 *ES 的demo
 */
public class test {

   //  private TransportClient client=null;

    /**
     * 高阶Rest Client
     */
    private RestHighLevelClient restHighLevelClient = null;

    @Before
    public void  getRestHighClient() throws IOException {
        //2.RestHighLevelClient与RestClient的创建：
        restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("10.20.10.162", 19200)));
        System.out.println("restHighLevelClient: "+restHighLevelClient);
        // restHighLevelClient.close();

    }

    @Test
    public void queryByTagLogs() {
       // getRestHighClient();
        ArrayList list = new ArrayList();
        // 1.创建并设置SearchSourceBuilder对象
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

        boolQueryBuilder.must(QueryBuilders.matchAllQuery());//查询所有 此处为匹配所有文档
        //elasticsearch 里默认的IK分词器是会将每一个中文都进行了分词的切割，所以你直接想查一整个词  加上.keyword
      //  boolQueryBuilder.must(QueryBuilders.termQuery("zz.keyword", "安徽省合肥市"));

        //时间范围的设定
        /*if (startTime != null && endTime != null){
            boolQueryBuilder.must(QueryBuilders.rangeQuery("time_file").from(startTime).to(endTime));

        }*/

        //把其它参数加进去，传过来的其他参数
       /* if (contextMap !=null){
            contextMap.forEach((k,v) ->{
                if (v !=null &&  "".equals(v)){
                    boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(k, v));
                }
            });
        }*/

        // 查询条件--->生成DSL查询语句
        searchSourceBuilder.query(boolQueryBuilder);
        //从第几页开始
        searchSourceBuilder.from(0);
        //每页显示几条
        searchSourceBuilder.size(100);
        //按照那一列排序，升降
        searchSourceBuilder.sort("id", SortOrder.ASC);

        // 2.创建并设置SearchRequest对象
        SearchRequest searchRequest = new SearchRequest();
        searchRequest.indices("acd_input_v1");//需要搜索的索引
        searchRequest.types("doc");//需要搜索的type
        // 设置SearchSourceBuilder查询属性
        searchRequest.source(searchSourceBuilder);

        //3.搜索数据
        try {

            SearchResponse search = restHighLevelClient.search(searchRequest);
            SearchHits hits = search.getHits();
            for (SearchHit hit :hits){
                /*Map sourceAsMap = hit.getSourceAsMap();
                String operateAt = sourceAsMap.get("operateAt").toString();
                SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
                long time = formatter.parse(operateAt).getTime();
                sourceAsMap.put("operateAt", time);

                list.add(sourceAsMap);

                Set> entrySet = sourceAsMap.entrySet();
                Iterator> it= entrySet.iterator();
                while (it.hasNext()){
                    Map.Entry me = it.next();
                     System.out.println(me.getKey()+"----------"+me.getValue());
                }*/
                 System.out.println(hit.getSourceAsString());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }/* catch (ParseException e) {
            e.printStackTrace();
        }*/

        //SearchRequestBuilder searchRequestBuilder = client.prepareSearch();
       // return list;
    }




    /**
     * 根据 id 获取数据
     * @throws Exception
     */
    @Test
    public void get() throws Exception{
        getRestHighClient();
        GetRequest request = new GetRequest("acd_input_v1", "doc", "5201020200700027");

        //可选参数设置
       /* request.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE);
        String[] includes = new String[]{"message", "*Date"};
        String[] excludes = Strings.EMPTY_ARRAY;
        FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
        request.fetchSourceContext(fetchSourceContext);*/

        //同步执行
        GetResponse getResponse = restHighLevelClient.get(request);

        //异步执行 listener的写法参照Index的异步执行的listener
        //restHighLevelClient.getAsync(request, listener);

        //Get Response 获取信息
        String sourceAsString = getResponse.getSourceAsString();
         System.out.println(sourceAsString);

    }

    /**
     * 查詢
     * @throws Exception
     */
    @Test
    public void search() throws Exception {
        //match all query 查询所有数据
        SearchRequest searchRequest = new SearchRequest();
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        //聚合函數 平均，最大，最小
//       AvgAggregationBuilder aggregation = AggregationBuilders.avg("avgnum").field("xq");
//        MaxAggregationBuilder aggre = AggregationBuilders.max("maxnum").field("xq");
        TermsAggregationBuilder termaggre = AggregationBuilders.terms("hphm").field("hphm").size(100);
        //      MinAggregationBuilder minaggre = AggregationBuilders.min("minnum").field("xq");
        searchSourceBuilder.query(QueryBuilders.matchAllQuery()).from(0);
        //  searchSourceBuilder.query(QueryBuilders.termQuery("bar2", "李亮")).size(100).from(5);//此处的size和from只是控制输出的数据条数，搜索的出来的记录数并不受控制
        //  searchSourceBuilder.query(QueryBuilders.rangeQuery("tjsj").gte("1000-08-13 15:31:26").lte("2019-08-13 15:31:26")).size(100).from(0);
        //组合聚合此处实在先分组，然后在每组里面获取最小值
        searchSourceBuilder.aggregation(termaggre);
        searchRequest.source(searchSourceBuilder);
        searchRequest.indices("veh_pass_v2");
        searchRequest.types("doc");


        //使用SearchSourceBuilder查询指定字段
//       SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//        sourceBuilder.query(QueryBuilders.termQuery("bar2", "李亮"));
//        sourceBuilder.from(0);
//        sourceBuilder.size(100);
//        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
//
//        SearchRequest searchRequest2 = new SearchRequest();
//        //index 数据库
//        searchRequest2.indices("acd_input_v1");
//        searchRequest2.types("doc");
//        searchRequest2.source(sourceBuilder);

        //Building queries
        //One way, QueryBuilder can be created using its constructor 使用QueryBuilder的构造函数
//        MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "kimchy");
//        matchQueryBuilder.fuzziness(Fuzziness.AUTO);
//        matchQueryBuilder.prefixLength(3);
//        matchQueryBuilder.maxExpansions(10);
        //Two way, QueryBuilder objects can also be created using the QueryBuilders utility class. 直接使用matchQuery
//        QueryBuilder matchQueryBuilder1 = matchQuery("user", "kimchy")
//                .fuzziness(Fuzziness.AUTO)
//                .prefixLength(3)
//                .maxExpansions(10);
//
//        searchSourceBuilder.query(matchQueryBuilder1);

        //Specifying Sorting 指定排序
//        sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
//        sourceBuilder.sort(new FieldSortBuilder("_uid").order(SortOrder.ASC));

        //Source filtering, turn off _source retrieval completely
//        sourceBuilder.fetchSource(false);
        //an array of one or more wildcard patterns to control which fields get included or excluded in a more fine grained way
//        String[] includeFields = new String[] {"title", "user", "innerObject.*"};
//        String[] excludeFields = new String[] {"_type"};
//        sourceBuilder.fetchSource(includeFields, excludeFields);

        //Requesting Aggregations
//        SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder();
//        TermsAggregationBuilder aggregation = AggregationBuilders.terms("by_company")
//                .field("company.keyword");
//        aggregation.subAggregation(AggregationBuilders.avg("average_age")
//                .field("age"));
//        searchSourceBuilder2.aggregation(aggregation);
//
//        //Requesting Suggestions
//        SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder();
//        SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("kmichy");
//        SuggestBuilder suggestBuilder = new SuggestBuilder();
//        suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder);
//        searchSourceBuilder3.suggest(suggestBuilder);

        //同步执行
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest);

        //异步执行 listener参考index的
//        restHighLevelClient.searchAsync(searchRequest, listener);

        //SearchResponse
        RestStatus status = searchResponse.status();
        for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
            // failures should be handled here
        }

        //Retrieving SearchHits 获取结果数据
        SearchHits hits = searchResponse.getHits();
//        long totalHits = hits.getTotalHits();
//        float maxScore = hits.getMaxScore();
        SearchHit[] searchHits = hits.getHits();
        //hits.getTotalHits() 是搜索出來的总记录数，不是返回的记录数，默认之返回10条
        System.out.println("getTotalHits:::::" + hits.getTotalHits());
        for (SearchHit hit : searchHits) {
            // do something with the SearchHit
            String index = hit.getIndex();
            String type = hit.getType();
            String id = hit.getId();
            float score = hit.getScore();

            String sourceAsString = hit.getSourceAsString();
            Map sourceAsMap = hit.getSourceAsMap();
//            String tjsj = (String) sourceAsMap.get("tjsj");


            System.out.println("sourceAsString:::::" + sourceAsString);
//             System.out.println("tjsj:"+tjsj);

        }
        //搜索的结果中返回的一些条数，默认最多是10条，具体看你上面设置的size大小。
        System.out.println("searchHits.length:::::" + searchHits.length);
        //普通的最大，最小，平均值的获取
        // Avg avg = searchResponse.getAggregations().get("maxnum");
        //  Max avg = searchResponse.getAggregations().get("maxnum");
       /* Min avg = searchResponse.getAggregations().get("minnum");
          double value = avg.getValue();
         System.out.println(value);*/


        //Retrieving Aggregations
        Map aggMap = searchResponse.getAggregations().asMap();
        Terms teamAgg = (Terms) aggMap.get("hphm");
        Iterator teamBucketIt = teamAgg.getBuckets().iterator();
        int size = teamAgg.getBuckets().size();
         System.out.println("teamAgg.length==="+size);
        while (teamBucketIt.hasNext()) {
            Terms.Bucket buck = teamBucketIt.next();

            Object key = buck.getKey();
            String keyAsString = buck.getKeyAsString();
            long count = buck.getDocCount();
            System.out.println("key:" + key.toString() + "--count:" + count);
             System.out.println("keyAsString:"+keyAsString);
//           //得到所有子聚合
//            Map subaggmap = buck.getAggregations().asMap();
//          //avg值获取方法
//            double amin_age= ((Min) subaggmap.get("minnum")).getValue();
//             System.out.println("min============"+amin_age);
            //sum值获取方法
            //  double total_salary = ((InternalSum) subaggmap.get("total_salary")).getValue();
        }
    }

        //Retrieving Suggestions
      /*  Suggest suggest = searchResponse.getSuggest();
        TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user");
        for (TermSuggestion.Entry entry : termSuggestion.getEntries()) {
            for (TermSuggestion.Entry.Option option : entry) {
                String suggestText = option.getText().string();
            }
        }
        }*/

    /**
     * 关于时间的AggregationBuilders
     */
    @Test
      public  void  searchAggrationTime() throws IOException {
        //match all query 查询所有数据
        SearchRequest searchRequest = new SearchRequest();
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(QueryBuilders.rangeQuery("tjsj").gte("2018-01-01 00:00:00").lte("2018-12-01 00:00:00"));

       // boolQueryBuilder.must(QueryBuilders.matchAllQuery()).filter(QueryBuilders.matchQuery("bar2","李亮"));
       // searchSourceBuilder.query(QueryBuilders.rangeQuery("tjsj").gte("2018-01-01 00:00:00").lte("2018-12-01 00:00:00")).size(100).from(0);
       // boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("bar2","李亮"));
        //3.聚合
         DateHistogramAggregationBuilder agge = AggregationBuilders.dateHistogram("sj")
                .field("tjsj").dateHistogramInterval(DateHistogramInterval.MONTH).format("MM月").minDocCount(0);
        TopHitsAggregationBuilder topagg = AggregationBuilders.topHits("top");
        /* .minDocCount(0).order(BucketOrder.key(true));*/

         searchSourceBuilder.aggregation(agge.subAggregation(topagg)).size(0).from(0).query(boolQueryBuilder).sort("tjsj",  SortOrder.ASC);
         //聚合之后在过滤的结果数据hits数据
        searchSourceBuilder.postFilter(QueryBuilders.rangeQuery("tjsj").gte("2018-01-01 13:32:26").lte("2018-12-01 00:00:00"));

        searchRequest.source(searchSourceBuilder);
        searchRequest.indices("acd_input_v1");
        searchRequest.types("doc");


        SearchResponse search = restHighLevelClient.search(searchRequest);
         System.out.println("getTotalHits:"+search.getHits().getTotalHits());
        SearchHit[] hits = search.getHits().getHits();
         System.out.println("count:"+hits.length);

        for (SearchHit hit:hits){
            String sourceAsString = hit.getSourceAsString();
             System.out.println("sourceAsString:"+sourceAsString);
        }
        ParsedDateHistogram sjdate = search.getAggregations().get("sj");
        if(sjdate.getBuckets()!=null) {
            for (Histogram.Bucket bucket : sjdate.getBuckets()) {
                 System.out.println( bucket.getKeyAsString()+"---"+bucket.getDocCount());
            }

            DecimalFormat df = new DecimalFormat( "0");
            long a=1;
            long b=2;
            long c=3;
            double v = ((double) a / (a + b + c))*100;
              System.out.println(v);
            System.out.println(df.format(v)+"%");
        }


        /*List buckets= ((ParsedDateHistogram)search.getAggregations().get("sj")).getBuckets();
        for(Histogram.Bucket bucket : buckets){
             System.out.println("date="+bucket.getKeyAsString()+",count="+bucket.getDocCount());
        }*/



    }

    /** 分组聚合 求 每一天/每一个月/每一年 的某个时间段的记录总数
     * AggregationBuilders的filter
     */
    @Test
    public  void  searchAggrationRange() throws IOException {
        //match all query 查询所有数据
        SearchRequest searchRequest = new SearchRequest();
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(QueryBuilders.rangeQuery("tjsj").gte("2018-01-01 00:00:00").lte("2018-12-31 00:00:00"));

        //3.聚合

        FiltersAggregator.KeyedFilter[] filters=new FiltersAggregator.KeyedFilter[31];
        for (int i=0;i<31;i++ ){
            filters[i] = new FiltersAggregator.KeyedFilter("2018-3-"+(i+1),
                    QueryBuilders.rangeQuery("tjsj").gte("2018-03-"+(i+1)+" 16:00:00").lte("2018-03-"+(i+1)+" 23:59:59"));
        }

//        FiltersAggregator.KeyedFilter keyedFilter1 = new FiltersAggregator.KeyedFilter("2018-3-18",
//                QueryBuilders.rangeQuery("tjsj").gte("2018-03-18 16:00:00").lte("2018-03-18 23:59:59"));


        AggregationBuilder aggregation = AggregationBuilders.filters("vionum",filters);
        searchSourceBuilder.aggregation(aggregation);
        searchSourceBuilder.query(boolQueryBuilder).size(0).sort("tjsj",SortOrder.ASC);

        searchRequest.source(searchSourceBuilder);
        searchRequest.indices("acd_input_v1");
        searchRequest.types("doc");

        SearchResponse search = restHighLevelClient.search(searchRequest);
        System.out.println("getTotalHits:"+search.getHits().getTotalHits());
        SearchHit[] hits = search.getHits().getHits();
        System.out.println("count:"+hits.length);

        for (SearchHit hit:hits){
            String sourceAsString = hit.getSourceAsString();
            System.out.println("sourceAsString:"+sourceAsString);
        }
        List terms = ((Filters)search.getAggregations().get("vionum")).getBuckets();
        List list = new ArrayList<>();
        TreeMap treemap = new TreeMap<>();
        for(Filters.Bucket bucket : terms){
            System.out.println(bucket.getKeyAsString()+"--------"+bucket.getDocCount());

            list.add(bucket.getDocCount()+"");
           /* ParsedDateHistogram dateHistogram = bucket.getAggregations().get("sj");
            if(dateHistogram.getBuckets()!=null && dateHistogram.getBuckets().size()>0) {
                for (Histogram.Bucket bucket2 : dateHistogram.getBuckets()) {
                     System.out.println(bucket2.getKeyAsString()+"--------"+bucket2.getDocCount());
                      System.out.println(bucket.getKeyAsString());
                }
            }*/
        }

         System.out.println("最大值是："+Collections.max(list)+"，最小值是："+ Collections.min(list));
    }

    /**
     * 查询某个字段中字段值的基数（种类）单字段去重
     *AggregationBuilders.cardinality("userAgg").field("user.keyword"))
     * 多字段去重
     *Script script = new Script("doc['anrType'].values +'####'+ doc['anrMessage'].values");
     * TermsAggregationBuilder app = AggregationBuilders.terms("app").script(script).size(10000);
     * //用于统计每一项详细数据
     * CardinalityAggregationBuilder app = AggregationBuilders.cardinality("app").script(script).precisionThreshold(10000);
     * //用于统计有多少项
     */

    @Test
public  void  searchAggrationCardinality(){

        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(QueryBuilders.matchAllQuery());
        //单个字段
        //CardinalityAggregationBuilder agg = AggregationBuilders.cardinality("hphm").field("hphm");
        //多个字段
        Script script = new Script("doc['hphm'].values +'_'+ doc['hpzl'].values");
        CardinalityAggregationBuilder agg = AggregationBuilders.cardinality("hphm").script(script);
                String[] str= new String[]{"hphm","hpzl"};
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().fetchSource(str,null);
        searchSourceBuilder.aggregation(agg);
      //  searchSourceBuilder.aggregation(agg);
        searchSourceBuilder.query(boolQueryBuilder);
        try {

           SearchResponse search = restHighLevelClient.search(new SearchRequest().source(searchSourceBuilder).indices("veh_pass").types("doc"));

            Cardinality can = search.getAggregations().get("hphm");
                //返回该字段值的种类
             System.out.println(can.getName()+"--"+can.getValue()+"--"+can.getValueAsString()+"--"+can.getType());

            SearchHit[] hits = search.getHits().getHits();
            for (SearchHit searchHit: hits) {
                String sourceAsString = searchHit.getSourceAsString();
                 System.out.println("searchHit=="+sourceAsString);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

    }


    /**
     *   * 同期数据然后去重，俩层aggrate叠加获取结果
     */
      @Test
    public  void  searchAggrationCardinalityAndFilter(){

        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(QueryBuilders.matchAllQuery());
        //单个字段
        //CardinalityAggregationBuilder agg = AggregationBuilders.cardinality("hphm").field("hphm");
        //多个字段
        Script script = new Script("doc['hphm'].values +'_'+ doc['hpzl'].values");
        CardinalityAggregationBuilder agg = AggregationBuilders.cardinality("hphm").script(script);
        FiltersAggregator.KeyedFilter[] filters = new FiltersAggregator.KeyedFilter[3];
        filters[0] = new FiltersAggregator.KeyedFilter("2019-01-23" , QueryBuilders.rangeQuery("jgsj").gte("2019-01-23 10:36:23").lte("2019-01-23 10:41:23"));
        filters[1] = new FiltersAggregator.KeyedFilter("2019-01-24" , QueryBuilders.rangeQuery("jgsj").gte("2019-01-24 10:36:23").lte("2019-01-24 10:41:23"));
        filters[2] = new FiltersAggregator.KeyedFilter("2019-01-25" , QueryBuilders.rangeQuery("jgsj").gte("2019-01-25 10:36:23").lte("2019-01-25 10:41:23"));
        AggregationBuilder agg1 = AggregationBuilders.filters("dayCount", filters);


        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(agg1.subAggregation(agg));
        //  searchSourceBuilder.aggregation(agg);
        searchSourceBuilder.query(boolQueryBuilder);
        try {

            SearchResponse search = restHighLevelClient.search(new SearchRequest().source(searchSourceBuilder).indices("veh_pass").types("doc"));

//            Cardinality can = search.getAggregations().get("hphm");
//            //返回该字段值的种类
//            System.out.println(can.getName()+"--"+can.getValue()+"--"+can.getValueAsString()+"--"+can.getType());

          //  Map asMap = search.getAggregations().getAsMap();
            //  Filters dayCount = (Filters)asMap.get("dayCount");
            Filters asMap = (Filters)search.getAggregations().get("dayCount");
            List buckets = asMap.getBuckets();
           for (int i=0;i dateTotalBuckets = dateTotalHis.getBuckets();
        for(Histogram.Bucket bucket:dateTotalBuckets){
            String keyAsString = bucket.getKeyAsString().substring(5,7);
            long total=bucket.getDocCount();
            System.out.println(keyAsString +"-------"+total);
            Sum swrsCounStringt = bucket.getAggregations().get("swrsCount");
            Sum ssrsCountString = bucket.getAggregations().get("ssrsCount");
            Sum zjccssCountString = bucket.getAggregations().get("zjccssCount");
        }
    }

}