ES安装
- install es in docker
- my kibana
- log kibana
基础
- 基本命令
POST twitter/_doc
{
"user_name":"jim",
"content":"enjoy es"
}
POST twitter/_doc/1
{
"user_name":"jack",
"content":"enjoy kafka"
}
GET twitter/_search
# 查看自动类型类型绑定，做适当调整
GET twitter/_mapping
DELETE twitter
PUT twitter
{
"settings": {
"number_of_shards": 5,
"number_of_replicas": 2
},
"mappings": {
"properties": {
"user_name":{
"type": "text"
},
"content":{
"type": "text"
}
}
}
}
- 新建模板
PUT _template/canaan
{
"index_patterns":"canaan*",
"settings": {
"index.number_of_replicas": "1",
"index.number_of_shards": "5",
"index.translog.flush_threshold_size": "512mb", # default 512mb, flush
"index.translog.sync_interval": "60s", # default 5s, fsync to disk
"index.codec": "best_compression", # 数据压缩算法，影响存储时间
"analysis": {
"filter":{ # token filter
"ngram_auto_complete_filter":{
"type":"edge_ngram",
"min_gram":1,
"max_gram":15
},
"chinese_pinyin_auto_complete_filter" : {
"type" : "pinyin",
"keep_first_letter" : true,
"keep_full_pinyin" : false,
"keep_joined_full_pinyin": true,
"keep_none_chinese" : true,
"keep_original" : true,
"limit_first_letter_length" : 16,
"lowercase" : true,
"trim_whitespace" : true,
"keep_none_chinese_in_first_letter" : true
},
"pinyin_index_filter" : {
"type" : "pinyin",
"keep_first_letter" : true,
"keep_full_pinyin" : true,
"keep_joined_full_pinyin": true,
"keep_none_chinese" : true,
"keep_original" : false,
"limit_first_letter_length" : 16,
"lowercase" : true,
"trim_whitespace" : true,
"keep_none_chinese_in_first_letter" : true
}
},
"tokenizer": {
"ik_max_word": {
"type": "ik_max_word",
"use_smart": true
}
},
"analyzer":{
"chinese_pinyin_auto_complete_analyzer":{
"type":"custom",
"char_filter": [
"html_strip"
],
"tokenizer":"keyword",
"filter":[
"lowercase",
"chinese_pinyin_auto_complete_filter",
"ngram_auto_complete_filter"
]
},
"chinese_auto_complete_analyzer":{
"type":"custom",
"char_filter": [
"html_strip"
],
"tokenizer":"keyword",
"filter":[
"lowercase",
"ngram_auto_complete_filter"
]
},
"pinyin_index_analyzer":{
"type":"custom",
"char_filter": [
"html_strip"
],
"tokenizer":"keyword",
"filter":[
"lowercase",
"pinyin_index_filter"
]
}
}
}
}
}
- 新建索引&&别名
DELETE /canaan_goods_v1
PUT /canaan_goods_v1
{
"mappings": {
"_doc":{
"properties" : {
"id":{
"type":"long"
},
"preview_image":{
"type":"keyword"
},
"name" : {
"type": "text",
"analyzer": "ik_max_word",
"fields": {
"pinyin":{
"type":"text",
"analyzer": "pinyin_index_analyzer"
},
"hint": {
"type": "text",
"analyzer": "chinese_pinyin_auto_complete_analyzer"
}
}
},
"title" : {
"type":"keyword"
},
"create_time" : {
"type":"date"
},
"update_time" : {
"type":"date"
},
"comment" : {
"type":"keyword"
},
"deleted" : {
"type":"boolean"
},
"is_force_unshelf" : {
"type":"boolean"
},
"salable":{
"type":"boolean"
},
"status" : {
"type":"keyword"
},
"support_lebang_order":{
"type":"boolean"
},
"min_price" : {
"type":"double"
},
"max_price" : {
"type":"double"
},
"min_donate" : {
"type":"double"
},
"max_donate" : {
"type":"double"
},
"sale_begin_time" : {
"type":"date"
},
"sale_end_time" : {
"type":"date"
},
"supplier_id" : {
"type":"long"
},
"supplier_whole_name" : {
"type":"keyword"
},
"supplier_type" : {
"type":"keyword"
},
"supplier_tatus" : {
"type":"keyword"
},
"supplier_simple_tatus" : {
"type":"keyword"
},
"logistics_template_id" : {
"type":"long"
},
"logistics_template_name" : {
"type" : "keyword"
},
"sys_organization_id" : {
"type":"long"
},
"sys_organization_name" : {
"type":"keyword"
},
"category_id" : {
"type" : "long"
},
"category_name" : {
"type" : "keyword"
},
"second_level_category_id" : {
"type" : "long"
},
"second_level_category_name" : {
"type" : "keyword"
},
"first_level_category_id" : {
"type" : "long"
},
"first_level_category_name" : {
"type" : "keyword"
},
"scopes":{
"type":"keyword"
},
"activity_id":{
"type":"long"
},
"activity_min_price":{
"type":"double"
},
"activity_max_price":{
"type":"double"
},
"activity_min_donate":{
"type":"double"
},
"activity_max_donate":{
"type":"double"
},
"activity_start_time":{
"type":"date"
},
"activity_end_time":{
"type":"date"
},
"activity_type":{
"type":"keyword"
},
"activity_status":{
"type":"keyword"
}
}
}
},
"aliases": {
"canaan_goods": {
"is_write_index":true
}
}
}
搜索
- 商品搜索
GET canaan_goods/_search
{
"from": 0,
"size": 10,
"query": {
"bool": {
"must": [
{
"dis_max": {
"tie_breaker": 0,
"queries": [
{
"term": {
"name": {
"value": "黄桃",
"boost": 5
}
}
},
{
"match": {
"name": {
"query": "黄桃",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 4
}
}
},
{
"term": {
"name.pinyin": {
"value": "黄桃",
"boost": 2
}
}
},
{
"wildcard": {
"name.pinyin": {
"wildcard": "*黄桃*",
"boost": 1
}
}
}
],
"boost": 1
}
}
],
"filter": [
{
"bool": {
"filter": [
{
"terms": {
"status": [
"SHELFING"
],
"boost": 1
}
},
{
"terms": {
"scopes": [
"1",
"960",
"440304",
"440000"
],
"boost": 1
}
},
{
"term": {
"deleted": {
"value": false,
"boost": 1
}
}
},
{
"term": {
"salable": {
"value": true,
"boost": 1
}
}
},
{
"range": {
"sale_begin_time": {
"from": null,
"to": "2019-10-29T08:22:58.952Z",
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
},
{
"range": {
"sale_end_time": {
"from": "2019-10-29T08:22:58.952Z",
"to": null,
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
}
}
- 其他搜索
POST user_tags/_doc
{
"user":"cs jim",
"tag":["android","elasticsearch","java","canal"],
"age":30
}
POST user_tags/_doc
{
"user":"cs jack",
"tag":["kafak","java","canaan"],
"age":20
}
GET user_tags/_mapping
POST user_tags/_search
{
"query": {
"bool": {
"must": {
"match": {
"user": "cs"
}
},
"filter": {
"term": {
"tag": "java"
}
},
"must_not": {
"range": {
"age": {
"gte": 1,
"lte": 10
}
}
},
"should": [
{
"term": {
"tag": "wow"
}
},
{
"term": {
"tag": "elasticsearch"
}
}
]
}
}
}
GET user_tags/_search
{
"query": {
"dis_max": {
"queries": [
{
"match": {
"user": "jack"
}
},
{
"term": {
"body": "java"
}
}
],
"tie_breaker": 0.7
}
}
}
GET _search
{
"query": {
"constant_score": {
"filter": {
"match": {
"user": "cs"
}
},
"boost": 1.2
}
}
}
#其他搜索：exists/must_not/fuzzy/ids/prefix/range/regexp/term/tems/wildcard/
聚合
POST kibana_sample_data_flights/_search
{
"size": 1
}
// min, max, sum, avg
POST kibana_sample_data_flights/_search?size=0
{
"aggs": {
"avg_AvgTicketPrice": {
"avg":{
"field":"AvgTicketPrice",
"missing":0
}
}
}
}
GET kibana_sample_data_flights/_search
{
"size": 0,
"aggs": {
"stats_AvgTicketPrice": {
"stats": {
"field":"AvgTicketPrice",
"missing":0
}
}
}
}
POST /kibana_sample_data_flights/_search?size=0
{
"aggs": {
"OriginCityName_count": {
"cardinality": {
"field": "OriginCityName",
"missing":"N/A"
}
}
}
}
GET /kibana_sample_data_flights/_search?size=0
{
"aggs": {
"percentiles_AvgTicketPrice": {
"percentiles": {
"field":"AvgTicketPrice",
"missing":0,
"percents": [
10,
20,
30,
99,
99.99
]
}
}
}
}
GET /kibana_sample_data_flights/_search?size=0
{
"aggs": {
"percentile_ranks_AvgTicketPrice": {
"percentile_ranks": {
"field": "AvgTicketPrice",
"values": [
250,
1000
]
}
}
}
}
POST /kibana_sample_data_flights/_search?size=0
{
"aggs": {
"top_AvgTicketPrice": {
"terms": {
"field": "DestCountry",
"size": 5
},
"aggs": {
"top_hits_AvgTicketPrice": {
"top_hits": {
"sort": [
{
"AvgTicketPrice": {
"order": "desc"
}
}
],
"_source": {
"includes": ["DestCountry","AvgTicketPrice"]
},
"size": 2
}
}
}
}
}
}
POST /kibana_sample_data_flights/_search?size=0
{
"aggs": {
"value_count_DestCountry": {
"value_count": {
"field":"DestCountry"
}
},
"value_count_OriginCountry":{
"value_count":{
"field":"OriginCountry"
}
}
}
}
GET kibana_sample_data_flights/_search?size=0
{
"aggs": {
"cardinality_DestCountry_from_OriginCountry_IT": {
"filter":{
"term":{
"OriginCountry":"IT"
}
},
"aggs": {
"cardinality_DestCountry":{
"cardinality":{
"field":"DestCountry"
}
}
}
}
}
}
GET kibana_sample_data_flights/_search?size=0
{
"aggs": {
"value_count_DestCountry_from_OriginCountry_IT": {
"filter":{
"term":{
"OriginCountry":"IT"
}
},
"aggs": {
"value_count_DestCountry":{
"value_count":{
"field":"DestCountry"
}
}
}
}
}
}
POST kibana_sample_data_ecommerce/_search
GET kibana_sample_data_ecommerce/_mapping
GET kibana_sample_data_ecommerce/_search?size=0
{
"aggs": {
"order_bucket": {
"composite": {
"sources": [
{
"date": {
"date_histogram": {
"field": "order_date",
"interval": "day"
}
}
}
]
}
}
}
}
GET kibana_sample_data_ecommerce/_search?size=0
{
"aggs": {
"order_bucket": {
"composite": {
"sources": [
{
"date": {
"date_histogram": {
"field": "order_date",
"interval": "day"
}
}
}
]
}
}
}
}
GET kibana_sample_data_flights/_search?size=0
{
"aggs": {
"time_rangs": {
"range": {
"field": "AvgTicketPrice",
"keyed": true,
"ranges": [
{
"key": "cheap",
"to": 200
},
{
"from": 200,
"to": 400
},
{
"from": 400,
"to": 800
},
{
"from": 800
}
]
}
}
}
}
GET kibana_sample_data_flights/_search?size=0
{
"query": {
"term": {
"DestCountry": "IT"
}
},
"aggs": {
"global_value_count_dest": {
"global": {},
"aggs": {
"aggs_value_count_dest": {
"value_count": {
"field": "OriginWeather"
}
}
}
},
"clear_value_count_dest": {
"value_count": {
"field": "OriginWeather"
}
}
}
}
#
# CIDR mask
# 223.87.60.27/25
# 223.87.60.27/11111111 11111111 11111111 10000000
# 227.87.60.0
#
GET kibana_sample_data_logs/_search
{
"size": 0,
"aggs": {
"ip_ranges": {
"ip_range": {
"field":"ip",
"ranges":[
{"to":"100.0.0.0","key":"part-1"},
{"from":"100.0.0.0","key":"part-2"},
{"mask":"223.87.60.27/25"},
{"mask":"120.49.143.213/25"}
]
}
}
}
}
POST /kibana_sample_data_flights/_search?size=0
{
"aggs":{
"flight_time":{
"histogram":{
"field":"FlightTimeMin",
"interval":200,
"keyed":true
}
}
}
}
GET kibana_sample_data_ecommerce/_search?size=0
{
"aggs": {
"date_histogram_order_date": {
"date_histogram": {
"field": "order_date",
"interval": "day"
}
}
}
}
GET kibana_sample_data_ecommerce/_search?size=0
{
"aggs": {
"date_range_order_date": {
"date_range": {
"field": "order_date",
"format": "yyyy-MM-dd",
"ranges": [
{
"to":"now+1d/d",
"from":"now-1d/d"
}
]
}
}
}
}
POST kibana_sample_data_logs/_search?size=0
{
"aggs": {
"logs_over_time": {
"auto_date_histogram": {
"field":"timestamp",
"buckets":10,
"format":"yyyy-MM-dd"
}
}
}
}
POST kibana_sample_data_logs/_search?size=1
{
"aggs": {
"viewport": {
"geo_bounds": {
"field": "geo.coordinates"
}
}
}
}
POST kibana_sample_data_logs/_search?size=0
{
"aggs": {
"src_terms": {
"terms": {
"field": "geo.src"
},
"aggs": {
"centroid": {
"geo_centroid": {
"field":"geo.coordinates"
}
}
}
}
}
}
# distance_type: arc(default), or plane(faster, but inaccurate)
POST /kibana_sample_data_logs/_search?size=0
{
"aggs" : {
"geo_distance_coordinates" : {
"geo_distance" : {
"field" : "geo.coordinates",
"origin" : "40, -99",
"unit":"km",
"distance_type" : "plane",
"ranges" : [
{ "to" : 100 ,"key": "first_ring"},
{ "from" : 100, "to" : 300, "key": "second_ring" },
{ "from" : 300, "key": "third_ring" }
],
"keyed":true
}
}
}
}
# 不同的precision, 地图会被划分为不同数量的网格，越大，网格数越大，单个网格表示的面积越小
# https://wiki.openstreetmap.org/wiki/Zoom_levels
POST /kibana_sample_data_logs/_search?size=0
{
"aggregations": {
"zoomed-in": {
"filter": {
"geo_bounding_box": {
"geo.coordinates": {
"top_left": "45, 100",
"bottom_right": "35, 97"
}
}
},
"aggregations": {
"zoom1": {
"geohash_grid": {
"field": "geo.coordinates",
"precision": 8
}
}
}
}
}
}
其他能力
ingest
PUT _ingest/pipeline/grok_ingest
{
"description": "grok description",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"
]
}
}
]
}
PUT grok_index/_doc/1?pipeline=grok_ingest
{
"message":"55.3.244.1 GET /index.html 15824 0.043"
}
GET grok_index/_doc/1
see more processors
ingest+reindex
DELETE user_json_index
DELETE user_json_string_index
DELETE _ingest/pipeline/json_ingest_demo
PUT _ingest/pipeline/json_ingest_user
{
"description": "json ingest demo",
"processors": [
{
"json": {
"field": "string_source",
"target_field": "user"
}
}
]
}
PUT user_json_string_index/_doc/1
{
"string_source":"{\"name\":\"zcj\"}"
}
PUT user_json_string_index/_doc/2
{
"string_source":"{\"name\":\"ljf\"}"
}
PUT user_json_string_index/_doc/3
{
"string_source":"{\"name\":\"wh\"}"
}
GET user_json_string_index/_search
PUT user_json_index/_doc/4
{
"user":{
"name":"ljx"
}
}
POST _reindex
{
"source": {
"index": "user_json_string_index"
},
"dest": {
"index": "user_json_index",
"pipeline": "json_ingest_user"
}
}
GET user_json_index/_search
kibana
- dev tools
- index patterns
- discover
- visualize
- dashboard