elasticsearch查询去重

实现查询去重、分页

例如：实现依据qid去重，createTime排序

DSL：

 GET  /nb_luban_answer/_search
 {
   "query": {
     "match": {
       "status": 1
     }
   },
   "sort": [
     {
       "createTime": {
         "order": "desc"
       }
     }
   ],"aggs": {
     "qid": {
       "terms": {
         "field": "qid",
         "size": 10
       },"aggs": {
         "rated": {
           "top_hits": {
             "sort": [{
               "createTime": {"order": "desc"}
             }],
             "size": 1
           }
         }
       }
     }
   },
   "size": 0,
   "from": 0
 }

执行的结果：

 {
   "_shards": {
     "total": 4,
     "failed": 0,
     "successful": 4
   },
   "hits": {
     "hits": [],
     "total": 4,
     "max_score": 0
   },
   "took": 4,
   "timed_out": false,
   "aggregations": {
     "qid": {
       "doc_count_error_upper_bound": 0,
       "sum_other_doc_count": 0,
       "buckets": [
         {
           "rated": {
             "hits": {
               "hits": [
                 {
                   "_index": "nb_luban_answer",
                   "_type": "luban_answer",
                   "_source": {
                     "img": "{\r\n  \"img1\" : \"11111'>alert(11111)'\",\r\n  \"img2\" : \"2222'>alert(11111)'\",\r\n  \"img3\" : \"33333'>alert(11111)'\"\r\n}",
                     "pin": "motai869",
                     "createTime": "2017-03-09 00:00:00",
                     "id": "SN-0000001792-9-1",
                     "tableNameSuffix": "1",
                     "qid": "SN-0000001536-9-3",
                     "content": "888'>alert(11111)'",
                     "status": 1
                   },
                   "_id": "SN-0000001792-9-1",
                   "sort": [
                     1489017600000
                   ],
                   "_score": null
                 }
               ],
               "total": 2,
               "max_score": null
             }
           },
           "doc_count": 2,
           "key": "SN-0000001536-9-3"
         },
         {
           "rated": {
             "hits": {
               "hits": [
                 {
                   "_index": "nb_luban_answer",
                   "_type": "luban_answer",
                   "_source": {
                     "img": "{\r\n  \"img1\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n  \"img2\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n  \"img3\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\"\r\n}",
                     "pin": "motai869",
                     "createTime": "2017-03-03 00:00:00",
                     "id": "SN-0000001280-9-3",
                     "tableNameSuffix": "3",
                     "qid": "SN-0000000256-0-3",
                     "content": "hahaha",
                     "status": 1
                   },
                   "_id": "SN-0000001280-9-3",
                   "sort": [
                     1488499200000
                   ],
                   "_score": null
                 }
               ],
               "total": 1,
               "max_score": null
             }
           },
           "doc_count": 1,
           "key": "SN-0000000256-0-3"
         },
         {
           "rated": {
             "hits": {
               "hits": [
                 {
                   "_index": "nb_luban_answer",
                   "_type": "luban_answer",
                   "_source": {
                     "pin": "motai869",
                     "createTime": "2017-03-02 00:00:00",
                     "id": "SN-0000208128-8-1",
                     "tableNameSuffix": "1",
                     "qid": "SN-0000207872-9-3",
                     "content": "hahaha",
                     "status": 1
                   },
                   "_id": "SN-0000208128-8-1",
                   "sort": [
                     1488412800000
                   ],
                   "_score": null
                 }
               ],
               "total": 1,
               "max_score": null
             }
           },
           "doc_count": 1,
           "key": "SN-0000207872-9-3"
         }
       ]
     }
   }
 }

JAVA API:

 String indexName="nb_luban_answer";
         String typeName="luban_answer";

         AggregationBuilder aggregation =
                 AggregationBuilders
                         .terms("agg").field("qid")
                         .subAggregation(
                                 AggregationBuilders.topHits("top").addSort("createTime",SortOrder.DESC).setSize(1)
                         );

         SearchResponse sResponse = Tool.CLIENT.prepareSearch(indexName).setTypes(typeName)
                 .setQuery(QueryBuilders.matchQuery("status",1))
                 .addSort("createTime", SortOrder.DESC)
                 .addAggregation(aggregation)
                 .execute().actionGet();
         Terms agg = sResponse.getAggregations().get("agg");
         for (Terms.Bucket entry : agg.getBuckets()) {
             String key = String.valueOf(entry.getKey()) ;
             long docCount = entry.getDocCount();
             System.out.println("key:"+ key +" doc_count:"+ docCount);

             TopHits topHits= entry.getAggregations().get("top");
             for (SearchHit hit : topHits.getHits()){
                 System.out.println(" -> id: "+ hit.getId()+" createTime: "+hit.getSource().get("createTime"));
             }
         }

ES中所有数据：

参考：Elasticsearch 5.x 字段折叠的使用点击打开链接