elasticsearch查询去重

实现查询去重、分页

例如：实现依据qid去重，createTime排序

DSL：

GET  /nb_luban_answer/_search
{
  "query": {
    "match": {
      "status": 1
    }
  },
  "sort": [
    {
      "createTime": {
        "order": "desc"
      }
    }
  ],"aggs": {
    "qid": {
      "terms": {
        "field": "qid",
        "size": 10
      },"aggs": {
        "rated": {
          "top_hits": {
            "sort": [{
              "createTime": {"order": "desc"}
            }], 
            "size": 1
          }
        }
      }
    }
  }, 
  "size": 0,
  "from": 0
} 

执行的结果：

{
  "_shards": {
    "total": 4,
    "failed": 0,
    "successful": 4
  },
  "hits": {
    "hits": [],
    "total": 4,
    "max_score": 0
  },
  "took": 4,
  "timed_out": false,
  "aggregations": {
    "qid": {
      "doc_count_error_upper_bound": 0,
      "sum_other_doc_count": 0,
      "buckets": [
        {
          "rated": {
            "hits": {
              "hits": [
                {
                  "_index": "nb_luban_answer",
                  "_type": "luban_answer",
                  "_source": {
                    "img": "{\r\n  \"img1\" : \"11111'>'\",\r\n  \"img2\" : \"2222'>'\",\r\n  \"img3\" : \"33333'>'\"\r\n}",
                    "pin": "motai869",
                    "createTime": "2017-03-09 00:00:00",
                    "id": "SN-0000001792-9-1",
                    "tableNameSuffix": "1",
                    "qid": "SN-0000001536-9-3",
                    "content": "888'>'",
                    "status": 1
                  },
                  "_id": "SN-0000001792-9-1",
                  "sort": [
                    1489017600000
                  ],
                  "_score": null
                }
              ],
              "total": 2,
              "max_score": null
            }
          },
          "doc_count": 2,
          "key": "SN-0000001536-9-3"
        },
        {
          "rated": {
            "hits": {
              "hits": [
                {
                  "_index": "nb_luban_answer",
                  "_type": "luban_answer",
                  "_source": {
                    "img": "{\r\n  \"img1\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n  \"img2\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\",\r\n  \"img3\" : \"jfs/t3184/361/7557351412/2685/17143f65/58b91982N4e71a5d8.jpg\"\r\n}",
                    "pin": "motai869",
                    "createTime": "2017-03-03 00:00:00",
                    "id": "SN-0000001280-9-3",
                    "tableNameSuffix": "3",
                    "qid": "SN-0000000256-0-3",
                    "content": "hahaha",
                    "status": 1
                  },
                  "_id": "SN-0000001280-9-3",
                  "sort": [
                    1488499200000
                  ],
                  "_score": null
                }
              ],
              "total": 1,
              "max_score": null
            }
          },
          "doc_count": 1,
          "key": "SN-0000000256-0-3"
        },
        {
          "rated": {
            "hits": {
              "hits": [
                {
                  "_index": "nb_luban_answer",
                  "_type": "luban_answer",
                  "_source": {
                    "pin": "motai869",
                    "createTime": "2017-03-02 00:00:00",
                    "id": "SN-0000208128-8-1",
                    "tableNameSuffix": "1",
                    "qid": "SN-0000207872-9-3",
                    "content": "hahaha",
                    "status": 1
                  },
                  "_id": "SN-0000208128-8-1",
                  "sort": [
                    1488412800000
                  ],
                  "_score": null
                }
              ],
              "total": 1,
              "max_score": null
            }
          },
          "doc_count": 1,
          "key": "SN-0000207872-9-3"
        }
      ]
    }
  }
}

JAVA API:

String indexName="nb_luban_answer";
        String typeName="luban_answer";

        AggregationBuilder aggregation =
                AggregationBuilders
                        .terms("agg").field("qid")
                        .subAggregation(
                                AggregationBuilders.topHits("top").addSort("createTime",SortOrder.DESC).setSize(1)
                        );

        SearchResponse sResponse = Tool.CLIENT.prepareSearch(indexName).setTypes(typeName)
                .setQuery(QueryBuilders.matchQuery("status",1))
                .addSort("createTime", SortOrder.DESC)
                .addAggregation(aggregation)
                .execute().actionGet();
        Terms agg = sResponse.getAggregations().get("agg");
        for (Terms.Bucket entry : agg.getBuckets()) {
            String key = String.valueOf(entry.getKey()) ;
            long docCount = entry.getDocCount();
            System.out.println("key:"+ key +" doc_count:"+ docCount);

            TopHits topHits= entry.getAggregations().get("top");
            for (SearchHit hit : topHits.getHits()){
                System.out.println(" -> id: "+ hit.getId()+" createTime: "+hit.getSource().get("createTime"));
            }
        }

ES中所有数据：

参考：Elasticsearch 5.x 字段折叠的使用点击打开链接