ES java api 分组聚合，以及查询api（滚动）

package com.xhzw.electric.service.impl;

import com.alibaba.fastjson.JSON;
import com.xhzw.electric.es.ElectricMeter;
import com.xhzw.electric.service.ElasticSearchService;
import com.xhzw.electric.util.Constant;
import com.xhzw.electric.util.DateUtil;
import com.xhzw.electric.util.StringUtil;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.Stats;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.*;

@Slf4j
@Service
@AllArgsConstructor
public class ElasticSearchServiceImpl implements ElasticSearchService {
    private final RestHighLevelClient restHighLevelClient;

    /**
     * 创建索引
     *
     * @throws IOException
     */
    @Override
    public void createIndex(String indexName) throws IOException {
//        CreateIndexRequest createIndexRequest = new CreateIndexRequest("electric-meter");
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
        restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
    }

    @Override
    public boolean existIndex(String indexName) {
        boolean exists=false;
//        String indexName="test-electric-meter-hour-0311";
        try {
            GetIndexRequest indexRequest=new GetIndexRequest(indexName);

            exists = restHighLevelClient.indices().exists(indexRequest, RequestOptions.DEFAULT);
        }catch (IOException e){
            e.printStackTrace();
        }

        return exists;
    }

    @Override
    public List likeIndex(String likeStr) throws IOException {
        Response get = restHighLevelClient.getLowLevelClient().performRequest(new Request("GET", "_cat/indices/"+likeStr));
        get.getEntity();
        String resultJson = EntityUtils.toString(get.getEntity());

        return getAllIndex(resultJson);

    }
    /**
     * 模糊查询索引
     * @param likeStr
     * @return
     * @throws IOException
     */
    @Override
    public void deleteAllByIndexName(String indexName) throws Exception{
        SearchRequest searchRequest=new SearchRequest(indexName);

        SearchSourceBuilder sourceBuilder=new SearchSourceBuilder();
        sourceBuilder.query();
        sourceBuilder.size(5000);
        searchRequest.source(sourceBuilder);
        SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        SearchHit[] hits = response.getHits().getHits();


        BulkRequest request = new BulkRequest();
        for (SearchHit hit : hits) {
            DeleteRequest deleteRequest=new DeleteRequest(indexName,hit.getId());
            request.add(deleteRequest);

        }
        restHighLevelClient.bulk(request,RequestOptions.DEFAULT);

    }
    /**
     * 插入数据
     * @param indexName
     * @param list
     * @throws IOException
     */
    @Override
    public void insertDate(String indexName,List list) throws IOException {
        if(list.size()==0){
            return;
        }
        //批量操作的Request
        BulkRequest request = new BulkRequest();
//        request.timeout("1s");
        //批量处理请求
        for (int i = 0; i < list.size(); i++) {
            request.add(
                    new IndexRequest(indexName)
                            .source(JSON.toJSONString(list.get(i)), XContentType.JSON)
            );
        }
        BulkResponse response = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
    }
    /**
     * 查询 and
     * @param indexName
     * @param map    列名，值
     * @return
     * @throws IOException
     */
    @Override
    public SearchHit[] searchLikeOne(String indexName,Map map) throws IOException{
        SearchRequest request = new SearchRequest(indexName);
        //构建搜索条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        map.entrySet().forEach(param->{
            boolQueryBuilder.must(QueryBuilders.termQuery(param.getKey(), param.getValue()));
        });

        sourceBuilder.query(boolQueryBuilder);
        request.source(sourceBuilder);
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        SearchHit[] hits = response.getHits().getHits();

        return hits;
    }

    /**
     * 查询区间段group聚合值
     * @param from 开始时间戳
     * @param to   结束时间戳
     * @param indexName   索引名称
     * @param group  计算分组
     * @param sumGroup 累计字段
     * @param sum   是否累加 或算差值
     * @return
     */
    @Override
    public Map getStatsGroupByDeviceId(long from, long to, List indexName, String[] group, String [] sumGroup, boolean sum){
        Map result=new HashMap<>();
        SearchRequest searchRequest=new SearchRequest(indexName.toArray(new String[indexName.size()]));
        SearchSourceBuilder sourceBuilder=new SearchSourceBuilder();
        sourceBuilder.query(QueryBuilders.boolQuery()
//                .must(QueryBuilders.rangeQuery("timestamp").gte(Timestamp.valueOf("2021-03-11 01:00:00").getTime()))
                        .must(QueryBuilders.rangeQuery("gather_time").gte(from).lte(to))
        );
        sourceBuilder.sort("gather_time", SortOrder.DESC);
        sourceBuilder.size(0);
//        String[] includes=new String[]{"timestamp","device_id","pf","ep"};
//        sourceBuilder.fetchSource(includes,null);
        TermsAggregationBuilder aggregation = AggregationBuilders.terms("devices")
                .field("device_id");
        Arrays.stream(group).forEach(type->{
            aggregation.subAggregation(AggregationBuilders.stats(type+"_stats")
                    .field(type));
        });
        sourceBuilder.aggregation(aggregation);
        searchRequest.source(sourceBuilder);
        try {
            SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            Aggregations aggregations = response.getAggregations();
            Terms byCompanyAggregation = aggregations.get("devices");
            List buckets = byCompanyAggregation.getBuckets();

            for (Terms.Bucket bucket : buckets) {
                String deviceId=(String)bucket.getKey();
                Map metersMap=new HashMap<>();
                metersMap.put("device_id",deviceId);
                Arrays.stream(group).forEach(type->{
                    Stats stats = bucket.getAggregations().get(type+"_stats");
                    metersMap.put(type,stats.getAvgAsString());
                    if(Arrays.asList(sumGroup).contains(type)){
                        if(sum){
                            metersMap.put(type,stats.getSum());
                        } else {
                            metersMap.put(type,stats.getMax()-stats.getMin());

                        }
                    }
                });
//                metersMap.put("timestamp",to);
                ElectricMeter electricMeter = JSON.parseObject(JSON.toJSONString(metersMap), ElectricMeter.class);
                result.put(deviceId,electricMeter);
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
        return result;
    }
    /**
     * 查询表最大时间
     * @return 0 /timestamp
     */
    public long getlastHourTime(String indexHourName){
        long result=0;
        try {
            SearchRequest searchRequest=new SearchRequest(indexHourName);
            SearchSourceBuilder sourceBuilder=new SearchSourceBuilder();

            TermsAggregationBuilder aggregation = AggregationBuilders.terms("timestamp")
                    .field("gather_time")
                    .size(1)
                    .order(BucketOrder.key(false));

            sourceBuilder.query();
            sourceBuilder.aggregation(aggregation);
            searchRequest.source(sourceBuilder);
            SearchResponse response = restHighLevelClient.search(searchRequest,RequestOptions.DEFAULT);

            Aggregations aggregations = response.getAggregations();
            Terms byCompanyAggregation = aggregations.get("timestamp");
            List buckets = byCompanyAggregation.getBuckets();
            for (Terms.Bucket bucket : buckets) {
                result=(long)bucket.getKey();
            }
        }catch (Exception e){
            log.error("查询最后一小时，不存在该表，返回0已做校验");
        }


        return result;
    }
    /**
     * 批查询
     * @param indexes  索引多表
     * @param fields   fields 判断 列的值
     * @param group    group不为null时:判断某一时刻同时都有值
     * @return
     */
    public List multiSearch(List indexes,Map> fields,String[] group) {
        List result = new ArrayList<>();

        MultiSearchRequest request = new MultiSearchRequest();

        Set>> entries = fields.entrySet();
        entries.forEach(entry ->{
            String field = entry.getKey();
            List values = entry.getValue();
            values.forEach( value ->{
                SearchRequest searchRequest = new SearchRequest(indexes.toArray(new String[indexes.size()]));
                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

                if(group!=null){
                    //必须同时存在值
                    Arrays.stream(group).forEach(type->{
                        boolQueryBuilder.must(QueryBuilders.existsQuery(type));
                    });
//                    //值不能同时为0
//                    BoolQueryBuilder boolQueryBuilderZero = QueryBuilders.boolQuery();
//                    Arrays.stream(group).forEach(type->{
//                        boolQueryBuilderZero.must(QueryBuilders.termQuery(type,"0"));
//                    });
//                    boolQueryBuilder.mustNot(boolQueryBuilderZero);
                }

                boolQueryBuilder.must(QueryBuilders.termQuery(field, value));
                searchSourceBuilder.query(boolQueryBuilder);

                searchSourceBuilder.size(1);
                searchSourceBuilder.sort("gather_time",SortOrder.DESC);
                searchRequest.source(searchSourceBuilder);
                request.add(searchRequest);
            });
        });
        try {
            MultiSearchResponse response = restHighLevelClient.msearch(request, RequestOptions.DEFAULT);
            response.forEach(t -> {
                SearchResponse resp = t.getResponse();
                SearchHit[] hits = resp.getHits().getHits();
                if(hits!=null && hits.length>0){
                    result.add(hits[0]);
                }

            });
        } catch (IOException e) {
            e.printStackTrace();
        }
        return result;
    }

    /**
     * 查询所有devices的最新一条同时具有group的数据集合
     * @param indexName 需要有租户id及当天 5-electric-meter-20210323
     * @param devices  所需要哪些deviceId的数据
     * @param group  同时存在哪些字段
     * @return
     */
    public Map getLastTimeValGroupByDeviceId(String indexName,List devices,String[] group){
        String yesterIndex = indexName.substring(0,indexName.lastIndexOf("-")+1) + DateUtil.dateDaySub1(indexName.substring(indexName.lastIndexOf("-")+1));
        //返回结果
        Map result=new HashMap<>();

        try {
                List indexes= new ArrayList<>();
                indexes.add(indexName);
                indexes.add(yesterIndex);

                Map> map = new HashMap<>();
                map.put("device_id",devices);
                List searchHits = multiSearch(indexes, map, Constant.I.group_i_single);
                searchHits.forEach( hit ->{
                    Map sourceMap = hit.getSourceAsMap();
                    ElectricMeter electricMeter = JSON.parseObject(JSON.toJSONString(sourceMap), ElectricMeter.class);
                    result.put(electricMeter.getDevice_id(),electricMeter);
                } );

        } catch (Exception e) {
            e.printStackTrace();
        }

        return result;
    }
    /**
     * 单设备查询
     * 查询deviceId同时具有groups中各个group exist的数据成一条记录
     * @param indexName 表名
     * @param deviceId  设备id
     * @param groups    同时存在哪些字段group集合
     * @return
     */
    public ElectricMeter getLastTimeValByDeviceId(String indexName,String deviceId,List groups){
        Map result= new HashMap<>();
        String yesterIndex = indexName.substring(0,indexName.lastIndexOf("-")+1) + DateUtil.dateDaySub1(indexName.substring(indexName.lastIndexOf("-")+1));
        List indexes= new ArrayList<>();
        indexes.add(indexName);
        if(existIndex(yesterIndex)){
            indexes.add(yesterIndex);
        }
        MultiSearchRequest request = new MultiSearchRequest();

        groups.forEach( group ->{
            SearchRequest searchRequest = new SearchRequest(indexes.toArray(new String[indexes.size()]));
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

            //必须同时存在值
            Arrays.stream(group).forEach(type->{
                boolQueryBuilder.must(QueryBuilders.existsQuery(type));
            });

            boolQueryBuilder.must(QueryBuilders.termQuery("device_id", deviceId));
            searchSourceBuilder.query(boolQueryBuilder);

            searchSourceBuilder.size(1);
            searchSourceBuilder.sort("gather_time",SortOrder.DESC);
            searchRequest.source(searchSourceBuilder);
            request.add(searchRequest);
        });
        try {
            MultiSearchResponse response = restHighLevelClient.msearch(request, RequestOptions.DEFAULT);
            int i = 0;
            for (MultiSearchResponse.Item item : response) {
                SearchResponse resp = item.getResponse();
                if(resp == null) continue;
                SearchHit[] hits = resp.getHits().getHits();
                if(hits.length > 0){
                    Arrays.stream(groups.get(i)).forEach(type->{
                        result.put(type,hits[0].getSourceAsMap().get(type));
                    });
                }
                i++;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }


        result.put("device_id",deviceId);
        return JSON.parseObject(JSON.toJSONString(result), ElectricMeter.class);
    }
    /**
     * 多设备查询汇总成一条记录
     * @param indexName 表名
     * @param ids  设备ids
     * @return
     */
    public ElectricMeter getLastestJoinMoreIds(String indexName,List ids){
        Map result = new HashMap<>();


        String yesterIndex = indexName.substring(0,indexName.lastIndexOf("-")+1) + DateUtil.dateDaySub1(indexName.substring(indexName.lastIndexOf("-")+1));
        List indexes= new ArrayList<>();
        indexes.add(indexName);
        if(existIndex(yesterIndex)){
            indexes.add(yesterIndex);
        }
        Map> map=new HashMap<>();
        map.put("device_id",ids);
        List searchHits = multiSearch(indexes, map, null);

        searchHits.forEach( hit ->{
            Map sourceMap = hit.getSourceAsMap();
            result.putAll(sourceMap);
        } );



        return JSON.parseObject(JSON.toJSONString(result), ElectricMeter.class);
    }

	public static List getAllIndex(String content){
		List result=new ArrayList<>();

		if(content.contains("\n")){
			Arrays.stream(content.split("\n")).forEach(t->{
				result.add(t.split(" ")[2]);
			});
		}
		return  result;
	}
}

public static void main(String[] args) {
        Map allDevice=new HashMap<>();

        String[] group={"ua","ub","uc"};

        String indexName="index-data-20210324";

        SearchRequest request=new SearchRequest(indexName);
        SearchSourceBuilder sourceBuilder=new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        
        //记录必须同时存在值
        Arrays.stream(group).forEach(type->{
            boolQueryBuilder.must(QueryBuilders.existsQuery(type));
        });
        //值不能同时为0
        BoolQueryBuilder boolQueryBuilderZero = QueryBuilders.boolQuery();
        Arrays.stream(group).forEach(type->{
            boolQueryBuilderZero.must(QueryBuilders.termQuery(type,"0"));
        });
        boolQueryBuilder.mustNot(boolQueryBuilderZero);

        //deviceId分组 取最新时间的size 1 时间倒序
        TermsAggregationBuilder aggregation = AggregationBuilders.terms("devices")
                .field("device_id");
        aggregation.subAggregation(AggregationBuilders.topHits("one").size(1).sort("gather_time", SortOrder.DESC));

        sourceBuilder.query(boolQueryBuilder);
        sourceBuilder.aggregation(aggregation);
        request.source(sourceBuilder);
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);

        Aggregations aggregations = response.getAggregations();
        Terms byCompanyAggregation = aggregations.get("devices");
        for(Terms.Bucket bucket:byCompanyAggregation.getBuckets()){
            // 获取name为one的top_hits结果集
            TopHits topHits = bucket.getAggregations().get("one");
            SearchHits hits = topHits.getHits();
            SearchHit[] hitArray = hits.getHits();
            // 因为top_hits的siez=1所以不进行遍历直接取第一条数据
            SearchHit hit = hitArray[0];
            Map sourceMap = hit.getSourceAsMap();
            //转对象
            ElectricMeter electricMeter = JSON.parseObject(JSON.toJSONString(sourceMap), ElectricMeter.class);
            allDevice.put(electricMeter.getDevice_id(),electricMeter);
        }
    }

滚动查询

List productCodeIdDTOS = new ArrayList<>();

        if (org.apache.commons.collections4.CollectionUtils.isEmpty(descTexts)){
            return productCodeIdDTOS;
        }

        BoolQueryBuilder builder = QueryBuilders.boolQuery();
        for (String descText : descTexts) {
            builder.should(QueryBuilders.wildcardQuery("descText.keyword", "*" + descText + "*"));
        }

        NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
                .withPageable(PageRequest.of(0,queryMax))
                .withSourceFilter(new FetchSourceFilter(new String[]{"id","productCode"},new String[0]))
                .withQuery(builder)

                .build();
        searchQuery.setTrackTotalHits(true);
        searchQuery.setMaxResults(5000);

        long scrollTimeOut = 60 * 1000;
        SearchHits result = elasticsearchRestTemplate.searchScrollStart(scrollTimeOut, searchQuery, ProductCodeIdDTO.class, IndexCoordinates.of(esIndexConfig.getBaseSpuIndexName()));

        int total = 0;
        while (result.hasSearchHits()) {
            long l = System.currentTimeMillis();
            System.out.println(total += result.getSearchHits().size());

            for (SearchHit searchHit : result.getSearchHits()) {
                ProductCodeIdDTO book = searchHit.getContent();
            }

//            // 再次查询
            result = elasticsearchRestTemplate
                    .searchScrollContinue(((SearchHitsImpl) result).getScrollId(), scrollTimeOut,
                            ProductCodeIdDTO.class, IndexCoordinates.of(esIndexConfig.getBaseSpuIndexName()));
            log.info("use time :{}",System.currentTimeMillis() - l);

        }

        List searchHitList = result.getSearchHits().stream().map(SearchHit::getContent).collect(Collectors.toList());