常用ES命令 Demo
push-trace-20190108/_search
{
"query": {
"bool": {
"must": [
{
"term": {
"event": "CONNECTOR_INSTANT_MESSAGE_DELIVER_SUCCESS"
}
},
{
"term": {
"appId": "otzoXoBSVbhEcUxAqYNdla3X"
}
}
]
}
},"sort": "eventTime"
}
{
"query": {
"bool": {
"must": [
{
"term": {
"appId": "SB4X6n005019ejXrqYVbRh3X"
}
},
{
"term": {
"event": "CONNECTOR_MESSAGE_DELIVER_SUCCESS"
}
}
]
}
},
"size": 0,
"aggs": {
"uniq_attr": {
"cardinality": {
"field": "messageId"
}
}
}
}
push_device_save_new/_search
{
"query": {
"bool": {
"must": [
{
"terms": {
"mcsVersion": [
"1.0.27","1.0.28","1.0.29","1.0.32","1.2.0","1.4.0","1.4.1","1.4.3"
],
"boost": 1
}
}
],
"disable_coord": false,
"adjust_pure_negative": true,
"boost": 1
}
},
"size": 0,
"timeout": "120s",
"_source": false,
"aggregations": {
"model": {
"terms": {
"field": "model",
"size": 200,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_term": "asc"
}
]
}
}
}
}
index模版参数及优化
{
"order": 0,
"template": "connector-connection*",
"settings": {
"index": {
"routing": {
"allocation": {
"total_shards_per_node": "5"
}
},
"refresh_interval": "300s",
"number_of_shards": "36",
"translog": {
"flush_threshold_size": "6GB",
"sync_interval": "60s",
"durability": "async"
},
"merge": {
"scheduler": {
"max_thread_count": "1"
}
},
"store": {
"throttle": {
"max_bytes_per_sec": "200mb"
}
},
"unassigned": {
"node_left": {
"delayed_timeout": "60m"
}
},
"number_of_replicas": "1"
}
},
"mappings": {
"type_push": {
"properties": {
"deviceId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"connectionTime": {
"type": "date",
"format": "epoch_millis"
},
"connectorIp": {
"type": "ip"
},
"clientIndex": {
"type": "keyword"
},
"mcsVersion": {
"type": "keyword"
},
"networkType": {
"type": "keyword"
}
}
}
},
"aliases": {}
}
curl语句
//更新，创建模板(匹配大小写问题)
curl -H 'Content-Type:application/json' -XPUT 10.15.74.16:9200/_template/push_device_save -d '{
"order": 0,
"template": "push_device_save",
"settings": {
"index": {
"analysis": {
"normalizer": {
"my_normalizer": {
"filter": [
"lowercase",
"asciifolding"
],
"type": "custom"
}
}
},
"number_of_shards": "9"
}
},
"mappings": {
"push_device_save": {
"properties": {
"id": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"imei": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"serialNumber": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"model": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"imsi": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"meid": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"ipMacAddress": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"ipv4": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"ipv6": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"wifiMacAddress": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"bluetoothMacAddress": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"androidId": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"androidVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"osVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"romVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"mcsVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"regionCode": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"otaVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"language": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"operator": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"basebandVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"kernelVersion": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"pcbNumber": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"brand": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"regionMark": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"persistentRegionMark": {
"type": "keyword",
"normalizer": "my_normalizer"
},
"createTime": {
"type": "date"
},
"updateTime": {
"type": "date"
}
}
}
},
"aliases": {}
}
'
//获取模板配置
curl -H 'Content-Type:application/json' -XGET 10.15.74.16:9200/_template/push_device_save
//修改批量拉取window大小
curl -H 'Content-Type:application/json' -XPUT http://10.35.208.15:9210/push_device_save/_settings -d '{ "index" : { "max_result_window" : 20000}}'
curl -H 'Content-Type:application/json' -XPUT http://10.35.208.15:9210/push_device_save_new/_search -d '{}'
//删除模板
curl -XDELETE 10.35.208.15:9210/push_device_save/push_device_save
//sroll 查询+分片slice
curl -H 'Content-Type:application/json' -X GET http://10.15.74.16:9200/_search?scroll=1m -d '{
"size" : 50000,
"timeout" : "90s",
"query" : {
"bool" : {
"must" : [
{
"terms" : {
"androidVersion" : [
"8.1.0"
],
"boost" : 1.0
}
},
{
"terms" : {
"model" : [
"OPPO R11"
],
"boost" : 1.0
}
},
{
"terms" : {
"mcsVersion" : [
"1.0.27"
],
"boost" : 1.0
}
}
],
"filter" : [
{
"range" : {
"updateTime" : {
"from" : "2019-04-28T16:00:00.000Z",
"to" : null,
"include_lower" : true,
"include_upper" : true,
"boost" : 1.0
}
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0
}
},
"_source":["id","imei"]
"sort" : [
{
"_doc" : {
"order" : "asc"
}
}
],
"slice" : {
"field" : "_uid",
"id" : 3,
"max" : 5
}
}'
curl -H 'Content-Type:application/json' -X GET http://10.1.8.46:9200/push_device_save_new/_search -d '{ "size": 0}'
curl -H 'Content-Type:application/json' -X GET http://10.1.15.43:9200/push_device_save_new/_search -d '{ "size": 0}'
//续写scroll正确scroll 第二类：不用index了 scroll 耗时需要什么
_search/scroll （cerebro里面不需要/开头）
{
"scroll":"5ms",
"scroll_id": "DnF1ZXJ5VGhlbkZldGNoCQAAAAAAfzD0FmtOc1FuSUpmUnpPVk9xYzRnU1Y5YncAAAAAAEARuRY0SXlFVU5ZQ1FJR2E1WTc2bFIxUjFRAAAAAAB_MPMWa05zUW5JSmZSek9WT3FjNGdTVjlidwAAAAAAQBG6FjRJeUVVTllDUUlHYTVZNzZsUjFSMVEAAAAAAH8w8hZrTnNRbklKZlJ6T1ZPcWM0Z1NWOWJ3AAAAAABAEbgWNEl5RVVOWUNRSUdhNVk3NmxSMVIxUQAAAAAAfzD2FmtOc1FuSUpmUnpPVk9xYzRnU1Y5YncAAAAAAEARuxY0SXlFVU5ZQ1FJR2E1WTc2bFIxUjFRAAAAAAB_MPUWa05zUW5JSmZSek9WT3FjNGdTVjlidw=="
}
控制台语句
//普通查询：综合query,filter,mustnot,source
index/_search
{
"from": 0,
"size": 50,
"query": {
"bool": {
"must": [
{
"terms" : {
"model" : [
"HD1901"
],
"boost" : 1.0
}
}
],
"filter" : [
{
"range" : {
"updateTime" : {
"from" : "2019-02-13T16:00:00.000Z",
"to" : null,
"include_lower" : true,
"include_upper" : true,
"boost" : 1.0
}
}
}
],
"must_not" : [
{
"terms" : {
"imei" : [
"10111"
],
"boost" : 1.0
}
}
]
}
},
"sort": [
{
"_doc": {
"order": "asc"
}
}
],
"_source" : false
}
//es 新增 ，没有_index,type
push_device_save/create {
"id":"1111",
"imei":"2222"
}
//es 更新，这样覆盖全部数据
push_device_save/push_device/591bedba563569631a87980c index/type/id
{} 直接更新数据
//es 更新，局部更新
push_device_save/push_device/591bedba563569631a87980c/_update
{
"doc" : {
"imei" : "only update imei"
}
}
//根据查询删除
push_device_save/_delete_by_query
{
"query": {
"bool": {
"filter": [
{
"term": {
"id": "1111"
}
}
]
}
}
}
//自定义标签最复杂
{
"size": 100,
"timeout": "20s",
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"terms": {
"imei": [
"22",
"44"
]
}
}
],
"must": [
{
"terms": {
"imei": [
"11"
]
}
}
],
"must_not": [
{
"terms": {
"imei": [
"55"
]
}
}
]
}
}
]
}
}
}
//聚类
{
"query": {
"bool": {
"must_not": [
{
"terms": {
"brand": [
"oppo",
"realme",
"oneplus"
],
"boost": 1
}
}
],
"disable_coord": false,
"adjust_pure_negative": true,
"boost": 1
}
},
"size": 0,
"timeout": "120s",
"_source": false,
"aggregations": {
"model": {
"terms": {
"field": "model",
"size": 200,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_term": "asc"
}
]
}
}
}
}
//es 删除数组操作
{
"script" : {
"source": "if (ctx._source.appId.contains(params.tag)) { ctx._source.appId.remove(ctx._source.appId.indexOf(params.tag)) }",
"lang": "painless",
"params" : {
"tag" : "ZngnvJIM7wQusNtbqYnpH6XX"
}
}
}
//es 新增数组操作
{
"script": {
"inline": "ctx._source.imei.add(params.new_imei)",
"params": {
"new_imei": "11"
}
}
}
//根据某个字段去重
curl -H 'Content-Type:application/json' -XGET http://10.15.74.16:9200/push_device_save_new/_search -d '{
"size":0,
"timeout": "120s",
"aggs":{
"count": {
"cardinality": {
"field": "imei"
}
}
}
}'
//根据某个字段去重
curl -H 'Content-Type:application/json' -XGET http://10.15.74.16:9200/push_device_save_new/_search -d '{
"query": {
"bool": {
"must": [
{
"terms": {
"mcsVersion": [
"2.0.0","2.0.1","2.0.2","2.0.3","2.0.4","2.0.5","2.0.6","2.0.7","2.0.8","2.0.9","2.0.10","2.1.0","2.1.1","2.1.2","2.1.3","2.1.4","2.1.5","2.1.6","2.1.7","2.1.8","2.1.9","2.1.10","2.2.0","2.2.1","2.2.2","2.2.3","2.2.4","2.2.5","2.2.6","2.2.7","2.2.8","2.2.9","2.2.10" ],
"boost": 1
}
}
],
"must" : [
{
"terms" : {
"language" : [
"th"
],
"boost" : 1.0
}
}
],
"disable_coord": false,
"adjust_pure_negative": true,
"boost": 1
}
},
"size":0,
"timeout": "120s",
"aggs":{
"count": {
"cardinality": {
"field": "DUID"
}
}
}
}'
//查询条件+聚类
{
"query": {
"bool": {
"must": [
{
"terms": {
"model": [
"cph1819"
],
"boost": 1
}
},
{
"terms" : {
"language" : [
"th"
],
"boost" : 1.0
}
}
],
"disable_coord": false,
"adjust_pure_negative": true,
"boost": 1
}
},
"size": 0,
"timeout": "120s",
"_source": false,
"aggregations": {
"model": {
"terms": {
"field": "mcsVersion",
"size": 200,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"order": [
{
"_count": "desc"
},
{
"_term": "asc"
}
]
}
}
}
}
//es update_by_query
curl -H 'Content-Type:application/json' -XPOST http://10.11.106.41:9201/push_device_save_new/_update_by_query -d '
{
"query": {
"bool": {
"filter": [
{
"term": {
"model": "pclm10"
}
}
],
"must_not" : [
{
"terms" : {
"brand" : [
"oppo",
"realme",
"oneplus"
],
"boost" : 1.0
}
}
]
}
},
"script": {
"source": "ctx._source['brand']='oppo'"
}
}'