elasticsearch 去重
ES字段折叠（collapse）可以去重，但是不hits里面的总数是不去重的，而聚合的cardinality方法去重，会有误差，默认30000之内的数据没有误差，参数precisionThresholdecisionThreshold最大可以调到40000.
DSL语句
{
"query": {
"bool": {
"must": [
{
"term" : {
"invaliddata.keyword" : {
"value" : "0",
"boost" : 1.0
}
}
},
{
"wildcard": {
"customername.keyword": "*周秀芳*"
}
}
]
}
},
"aggs": {
"customerid": {
"cardinality": {
"field": "customerid.keyword"
}
}
},
"collapse":{
"field":"customerid.keyword",
"inner_hits": {
"name": "top_rated",
"size": 1
}
},
"size": 50
}
结果
{"took": 772,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 22,
"max_score": null,
"hits": [
{
"_index": "mtd_customers",
"_type": "mtd_customers_type",
"_id": "4B418955D0766B53E0530B0A420AC4DC_TACT",
"_score": 1.0744119,
"_source": {
"firstname": null,
"birthdate": "1980-10-15T16:00:00.000Z",
"occupation": null,
"identitynumber": "33022419801016",
"customercondition": "0",
"source": "TACT",
"credate": null,
"icropid": null,
"isttime": "2017-11-30T11:53:41.000Z",
"dealerspaceid": null,
"invaliddata": "0",
"customercagegory": null,
"nickname": null,
"respectcode": null,
"id": "4651281237",
"customername": "周秀芳",
"customertype": "1",
"buscleanflag": 1,
"incomamount": null,
"userfullname": null,
"identitytype": "1",
"sex": "2",
"deltime": null,
"middlename": null,
"createuser": null,
"lastname": null,
"ity": null,
"buytel": "135866",
"maritalstatus": "4",
"upttime": "2017-11-30T12:12:40.000Z",
"identitytext": null,
"customerid": "4B418955D0766B53E0530B0A420AC4DC",
"customerflag": 2,
"updatetime": "2017-06-22T14:13:41.000Z",
"corporatename": null,
"respectname": null,
"updateuser": null
},
"fields": {
"customerid.keyword": [
"4B418955D0766B53E0530B0A420AC4DC"
]
},
"inner_hits": {
"top_rated": {
"hits": {
"total": 2,
"max_score": 1.0744119,
"hits": [
{
"_index": "mtd_customers",
"_type": "mtd_customers_type",
"_id": "4B418955D0766B53E0530B0A420AC4DC_TACT",
"_score": 1.0744119,
"_source": {
"firstname": null,
"birthdate": "1980-10-15T16:00:00.000Z",
"occupation": null,
"identitynumber": "33022419801016",
"customercondition": "0",
"source": "TACT",
"credate": null,
"icropid": null,
"isttime": "2017-11-30T11:53:41.000Z",
"dealerspaceid": null,
"invaliddata": "0",
"customercagegory": null,
"nickname": null,
"respectcode": null,
"id": "4651281237",
"customername": "周",
"customertype": "1",
"buscleanflag": 1,
"incomamount": null,
"userfullname": null,
"identitytype": "1",
"sex": "2",
"deltime": null,
"middlename": null,
"createuser": null,
"lastname": null,
"ity": null,
"buytel": "13586653875",
"maritalstatus": "4",
"upttime": "2017-11-30T12:12:40.000Z",
"identitytext": null,
"customerid": "4B418955D0766B53E0530B0A420AC4DC",
"customerflag": 2,
"updatetime": "2017-06-22T14:13:41.000Z",
"corporatename": null,
"respectname": null,
"updateuser": null
}
}
]
}
}
}
}
,
"aggregations": {
"customerid": {
"value": 20
}
}
}
总数用aggregations中的value，结果列表用hits