Elasticsearch API操作01
基本操作
集群健康度检查
GET http://172.16.18.114:9200/_cluster/health判断索引是否存在
HEAD http://172.16.18.114:9200/logstash-2016.01.07创建索引
PUT http://172.16.18.116:9200/test
{
"settings" : {
"index" : {
"number_of_shards" : 3,
"number_of_replicas" : 2
}
}
}查看索引映射信息
GET http://172.16.18.116:9200/cdr-2015-08-30/_mappings修改索引映射
PUT http://172.16.18.116:9200/logstash-2015.11.17/heartbeat/_mappings
{
"_ttl": {
"enabled": true,
"default": "3d"
},
"properties": {
"message": {
"index": "no",
"type": "string"
},
"@timestamp": {
"format": "dateOptionalTime",
"type": "date"
},
"host": {
"index": "no",
"type": "string"
}
},
"_all": {
"enabled": false
}
}查看索引属性
GET http://172.16.18.114:9200/logstash-2015.12.30/_settings设置索引属性
PUT http://172.16.18.116:9200/.marvel-kibana/_settings
{
"index" : {
"number_of_replicas" : 0
}
}查看索引信息
GET http://172.16.18.116:9200/cdr-2015-08-30/_stats关闭索引(使用时再_open)
POST http://172.16.18.114:9200/logstash-2015.11*/_close列出所有节点简要状态信息
GET http://172.16.18.116:9200/_nodes/stats/indices/search查看节点配置情况
GET http://172.16.18.116:9200/_nodes/elasticsearch_114/settings列出节点存储信息
GET http://172.16.18.114:9200/_stats/store查看节点插件
GET http://172.16.18.114:9200/_nodes/elasticsearch_114/plugins?pretty=true列出节点详细信息
GET http://172.16.18.116:9200/_stats查询特定字段
POST http://172.16.18.116:9200/my_index/_search?fields=_all,_source,full_name
{
"query": {
"match": {
"_all": "John Smith"
}
},
"highlight": {
"fields": {
"_all": {},
"full_name": {}
}
}
}查询索引下数据
GET http://172.16.18.116:9200/kibana-int/_search查看某一索引/类型下文档总数
GET http://172.16.18.116:9200/cdr-2015-10-21/u2_gcdr/_count单词搜索
POST http://172.16.18.116:9200/jfyindex/_search
{
"query" : {
"match" : {
"_all": "rock climbing"
}
}
}简单搜索
GET http://172.16.18.116:9200/jfy*/_search短语搜索
GET http://172.16.18.116:9200/jfyindex/_search?q="rock climbing"短语搜索
POST http://172.16.18.116:9200/jfyindex/_search?q="rock climbing"
{
"query" : {
"match_phrase" : {
"_all": "rock climbing"
}
}
}搜索清单号码（全词匹配）
POST http://172.16.18.116:9200/cdr*/_search
{
"query" : {
"match" : {
"msisdn": "85267944762"
}
}
}分析查询文本
POST http://172.16.18.116:9200/logstash-2015.08.01/_analyze O_Str[9]=20150630-111344Explan Query
POST http://172.16.18.116:9200//_validate/query?explain
{
"query": {
"query_string": {
"query": "abc"
}
}
}定义模板
PUT http://172.16.18.116:9200/_template/cdr
{
"order": 0,
"template": "cdr*", "settings": { "index.refresh_interval": "30m", "index.number_of_replicas": "1", "index.number_of_shards": "1", "index.translog.flush_threshold_ops": "100000" }, "mappings": { "_default_": { "dynamic_templates": [ { "string_fields": { "mapping": { "index": "not_analyzed", "omit_norms": true, "type": "string" }, "match": "*",
"match_mapping_type": "string"
}
}
],
"_all": {
"enabled": true
},
"date_detection": false
},
"hjscdr": {
"properties": {
"starttime": {
"index": "not_analyzed",
"type": "string"
},
"msisdn": {
"index": "not_analyzed",
"type": "string"
},
"rectype": {
"index": "no",
"type": "string"
},
"pdp_address": {
"index": "not_analyzed",
"type": "string"
},
"roaming_charge": {
"type": "long"
},
"unit": {
"type": "long"
},
"totallink": {
"type": "long"
},
"@timestamp": {
"type": "long"
},
"startdate": {
"index": "not_analyzed",
"type": "string"
},
"charging_item": {
"index": "not_analyzed",
"type": "string"
},
"sgsn": {
"index": "not_analyzed",
"type": "string"
},
"imsi": {
"index": "not_analyzed",
"type": "string"
},
"apn": {
"index": "not_analyzed",
"type": "string"
},
"uplink": {
"type": "long"
},
"ggsn": {
"index": "not_analyzed",
"type": "string"
},
"downlink": {
"type": "long"
}
}
}
}
}logstash模板示例
PUT http://172.16.18.116:9200/_template/logstash
{
"order": 99,
"template": "logstash-*",
"settings": {
"index.refresh_interval": "5s",
"index.number_of_replicas": "1",
"index.number_of_shards": "1",
"index.translog.flush_threshold_ops": "5000"
},
"mappings": {
"_default_": {
"_all": {
"enabled": true,
"omit_norms": true
},
"dynamic_templates": [
{
"message_field": {
"match": "message",
"match_mapping_type": "string",
"mapping": {
"type": "string",
"index": "analyzed",
"omit_norms": true
}
}
},
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "string",
"index": "analyzed",
"omit_norms": true,
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"@version": {
"type": "string",
"index": "not_analyzed"
},
"geoip": {
"type": "object",
"dynamic": true,
"properties": {
"location": {
"type": "geo_point"
}
}
}
}
}
}
}
} 进阶操作
刷新数据到磁盘
POST http://172.16.18.116:9200/_flush设置索引刷新频率
PUT http://172.16.18.116:9200/cdr*/_settings
{
"index.translog.flush_threshold_ops": 5000,
"index.refresh_interval": "5s"
}查看node segment的memory
GET http://172.16.18.179:9200/_cat/nodes?v=&h=name,port,sm查看fielddata占用内存情况(查询时es会把fielddata信息load进内存)
GET http://172.16.18.114:9200/_stats/fielddataaggs max(求某个索引类型一段时间内某一字段最大值)
POST http://172.16.18.116:9200/logstash-2015.11.17/heartbeat/_search
{
"query": {
"range": {
"@timestamp": {
"from": 1547330883236,
"to": 1547330883236
}
}
},
"size": 0,
"aggs": {
"max_time": {
"max": {
"field": "@timestamp"
}
}
}
}监控FieldData
GET http://172.16.18.114:9200//_nodes/stats/indices/fielddata?fields=*cat fielddata
https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-fielddata.html
GET http://172.16.18.179:9200/_cat/fielddata?v多列聚合操作
实现如mysql的group by功能：
select yearmon(createtime) yearmon,method,status,count(*) from test group by yearmon,method,status
POST http://172.16.18.116:9200/test/_search
{
"size": 0,
"aggs" : {
"day_total" : {
"date_histogram":{
"field": "CreateTime",
"interval": "day",
"format": "yyyyMMdd"
},
"aggs": {
"method_total":{
"terms": {"field": "method"},
"aggs":{
"status_total":{
"terms": {"field": "status"}
}
}
}
}
}
}
}设置文档_ttl值
PUT http://172.16.18.116:9200/_template/template_logstash
{
"template" : "logstash*",
"settings" : {
"number_of_shards" : 1,
"number_of_replicas": 1
},
"mappings" : {
"hostapd.log":{
"_ttl" : { "enabled" : true, "default" : "30d" }
},
"hostapd1.log":{
"_ttl" : { "enabled" : true, "default" : "30d" }
}
}
}Indices shard stores(2.0以上)
GET http://172.16.18.114:9200//logstash-2016.01.07/_shard_stores节点jvm信息
GET http://172.16.18.114:9200/_nodes/*/stats/jvm暂停集群的shard自动均衡
PUT http://172.16.18.116:9200/_cluster/settings
{
"transient" : {
"cluster.routing.allocation.enable" : "none"
}
}恢复集群的shard自动均衡
PUT http://172.16.18.179:9200/_cluster/settings
{
"transient" : {
"cluster.routing.allocation.enable" : "all"
}
}重启节点
POST http://172.16.18.116:9200/_cluster/nodes/elasticsearch_114/_shutdown查看索引Segments信息
GET http://172.16.18.116:9200//logstash-2016.01.07/_segmentsscan-and-scroll
POST http://172.16.18.116:9200/_search/scroll?scroll=1m c2NhbjsxOzU3NzIzOmZpVXBnR0VDUl9lRFF3ejBJUjBkOVE7MTt0b3RhbF9oaXRzOjE5OTc2ODs=Limiting Memory Usage(fielddata-size)
https://www.elastic.co/guide/en/elasticsearch/guide/current/_limiting_memory_usage.html#fielddata-size
POST http://172.16.18.116:9200//_cluster/settings
{
"persistent" : {
"indices.fielddata.break.limit" : "80%",
"indices.fielddata.cache.size" : "60%"
}
}清理cache
POST http://172.16.18.114:9200//_cache/clear优化索引
POST http://172.16.18.114:9200/_optimize?max_num_segments=1nodes/hot_threads
GET http://172.16.18.114:9200/_nodes/hot_threads刷新synced(1.7以上才支持)
PUT http://172.16.18.116:9200/_flush/synced
{
"transient" : {
"cluster.routing.allocation.enable" : "none"
}
}段合并节流控制
PUT http://172.16.18.116:9200//_cluster/settings
{
"persistent" : {
"indices.fielddata.break.limit" : "80%",
"indices.fielddata.cache.size" : "60%"
}
}查看segment的memory
GET http://172.16.18.179:9200/_cat/segments?v深翻页(scan-and-scroll)
POST http://172.16.18.116:9200/test_index/_search?search_type=scan&scroll=1m
{
"query": {
"match_all": {}
},
"size": 1000
}集群性能优化
PUT http://172.16.18.116:9200//_cluster/settings
{
"persistent" : {
"indices.store.throttle.type" : "none",
"indices.store.throttle.max_bytes_per_sec" : "100mb",
"indices.fielddata.break.limit" : "80%",
"indices.fielddata.cache.size" : "60%"
}
}searchguard-auth
PUT http://172.16.18.114:9201/searchguard/ac/ac
{"acl": [
{
"__Comment__": "Default is to execute all filters",
"filters_bypass": [],
"filters_execute": ["actionrequestfilter.deny"]
},
{
"__Comment__": "This means that every requestor (regardless of the requestors hostname and username) which has the root role can do anything",
"roles": ["root"],
"filters_bypass": ["*"],
"filters_execute": []
},
{
"__Comment__": "172.16.18.171 can do anything",
"hosts": ["172.16.18.171"],
"filters_bypass": ["*"],
"filters_execute": []
},
{
"__Comment__": "172.16.18.114 readonly",
"hosts": ["172.16.18.114"],
"filters_bypass": ["actionrequestfilter.deny"],
"filters_execute": ["actionrequestfilter.readonly"]
},
{
"__Comment__": "172.16.18.114, index: *kibana*, can do anything",
"hosts": ["172.16.18.114"],
"indices": ["*kibana*"],
"filters_bypass": ["*"],
"filters_execute": []
},
{
"__Comment__": "This means that for the user spock on index popstuff only the actionrequestfilter.readonly will be executed, no other",
"users": ["user"],
"indices": ["cdr*"],
"filters_bypass": ["actionrequestfilter.deny"],
"filters_execute": ["actionrequestfilter.readonly"]
}
]}