Elasticsearch使用ik分词、go-mysql-elasticsearch同步工具使用以及es精准映射相关操作

一、安装ik分词器

• 获取ik分词器安装包

ik分词器的版本必须和elasticsearch版本一致
下载安装包 wget https://github.com/medcl/elasticsearch-analysis-ik/releases/tag/v7.12.1

• 解压到宿主机映射的es扩展目录

 unzip /home/elasticsearch-analysis-ik-7.12.1.zip -d /docker/es/plugins/ik/

二、同步mysql工具 go-mysql-elasticsearch

#拉取 gozer/go-mysql-elasticsearch 镜像
pull gozer/go-mysql-elasticsearch
#
mkdir /docker/go-mysql-es #创建宿主机映射目录
cd go-mysql-es/
touch go_mysql_river.toml
vim go_mysql_river.toml 

#内容如下
my_addr = "172.17.0.5:3306"    #mysql地址与端口
my_user = "test"               #mysql用户
my_pass = "123456"             #mysql密码
my_charset = "utf8"            #字符集
enable-relay = true            #mysql中继日志获取

es_addr = "172.17.0.8:9200"    #es的地址与端口
es_user = ""                   #es用户名
es_pass = ""                   #es密码

data_dir = "/docker/data"      #数据存储目录

stat_addr = "127.0.0.1:12800"  #默认配置
stat_path = "/metrics"

server_id = 1001               #不能与mysql配置中的server_id冲突
flavor = "mysql"               #连接的数据库类型
mysqldump = ""                 #不配置默认增量新增（mysql8.0+配置后运行会有异常）

#skip_master_data = false
bulk_size = 128                --
flush_bulk_time = "200ms"
skip_no_pk_table = false

[[source]]
schema = "lmrs"     #同步的数据库
tables = ["lmrs_products"]     #同步的表

[[rule]]
schema = "lmrs"     		   #同步的数据库
table = "lmrs_products"        #同步的表
index = "pro_list"             #同步的es索引
type = "_doc"                  #同步的es类型
# Only sync following columns
filter = ["id", "name","long_name","brand_id","shop_id","price","sold_count","review_count","status","create_time","last_time","three_category_id"]  #哪些字段同步
[rule.filed]
mysql = "three_category_id"  #mysql的three_category_id替换es的category_id
elastic = "category_id"

#构建容器
docker run -p 12345:12345 -d --name go-mysql-es -v /docker/go-mysql-es/go_mysql_river.toml:/config/river.toml:ro --privileged=true gozer/go-mysql-elasticsearch

#遇到此报错 binlog must ROW format, but MIXED now
mysql binlog必须是ROW模式
修改/etc/my.cnf binlog_format=row

• 查看运行同步日志
  

三、Elasticsearch 字段类型

• 字符串类型：
  （1）text：通常用来存储长文本字段，例如博客标题、内容。通过定义analyzer，实现分词索引。
  （2）keyword：不分词的字符串类型，存什么就是什么。
• 数值类型：long、integer、short、byte、double、float
• 日期类型：date
• 布尔类型：boolean
• 复合类型：
  （1）object：包含一个子对象
  （2）nested：包含一个或多个子对象
• 地理类型：geo_point
• 其他特定类型：ip、completion、join、alias等等

四、精确映射

动态映射>>

当我们需要自己控制字段类型，而不需要es凭借猜测自动映射，此时就可以使用精确映射。也就是提前创建好mapping，然后再添加数据，这也是目前大多数项目中使用的方式。

1. 创建user索引

PUT /user/
{
  "mappings": {
    "properties": {
      "user_name":{
        "type": "text",
        "analyzer": "ik_smart"
      },
      "age":{
        "type": "integer"
      }
    }
  }
}

2. 给索引添加字段(对user索引添加一个名称为create_at的字段，类型是date)

PUT /user/_mapping
{
  "properties":{
    "create_at":{
      "type": "date",
      "format": "yyyy-MM-dd HH:mm:ss"
    }
  }
}

elasticsearch不支持更改现有字段的映射或字段类型。

3. 查看索引的映射mapping

GET /user/_mapping
#结果如下
{
  "user" : {
    "mappings" : {
      "properties" : {
        "age" : {
          "type" : "integer"
        },
        "create_at" : {
          "type" : "date",
          "format" : "yyyy-MM-dd HH:mm:ss"
        },
        "user_name" : {
          "type" : "text",
          "analyzer" : "ik_smart" #使用ik
        }
      }
    }
  }
}
##---查看指定映射
GET /user/_mapping/field/age
#结果如下
{
  "user" : {
    "mappings" : {
      "age" : {
        "full_name" : "age",
        "mapping" : {
          "age" : {
            "type" : "integer"
          }
        }
      }
    }
  }
}

五、常用查询语法

1. GET /products/_search
#查询结果
{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 4,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "1",
        "_score" : 1.0,
        "_source" : {
          "user_name" : "zero",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "2",
        "_score" : 1.0,
        "_source" : {
          "user_name" : "zero1",
          "age" : 20,
          "create_at" : "2021-10-25 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "3",
        "_score" : 1.0,
        "_source" : {
          "user_name" : "zero error",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "4",
        "_score" : 1.0,
        "_source" : {
          "user_name" : "zero two",
          "age" : 20,
          "create_at" : "2021-10-25 12:00:00"
        }
      }
    ]
  }
}

2. 普通查询

GET /user/_doc/_search
{
  "query":{
    "match":{
      "user_name":"zero 1"
    }
  }
}
#结果如下
#! [types removal] Specifying types in search requests is deprecated.
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 3,
      "relation" : "eq"
    },
    "max_score" : 0.41299206,
    "hits" : [
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "1",
        "_score" : 0.41299206,
        "_source" : {
          "user_name" : "zero",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "3",
        "_score" : 0.31387398,
        "_source" : {
          "user_name" : "zero error",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "4",
        "_score" : 0.31387398,
        "_source" : {
          "user_name" : "zero two",
          "age" : 20,
          "create_at" : "2021-10-25 12:00:00"
        }
      }
    ]
  }
}

3. 指定字段（user_name）查询

GET /user/_doc/_search
{
  "query":{
    "match":{
      "user_name":"zero 1"
    }
  },
  "_source":["user_name"]
}

4. 根据字段（age desc）排序查询

GET /user/_doc/_search
{
  "query":{
    "match":{
      "user_name":"zero 1"
    }
  },
  "_source":["user_name","age"],
   "sort":[
    {
      "age":{
        "order":"desc"
      }
    }
  ]
}

5. 分页查询

GET /user/_doc/_search
{
  "query":{
    "match":{
      "user_name":"zero 1"
    }
  },
  "_source":["user_name","age"],
   "sort":[
    {
      "age":{
        "order":"desc"
      }
    }
  ],
  "from":0, #页码从0开始
  "size":1 #每页分页数量
}

6. 布尔查询，多条件查询

• must（and），所有的条件都要符合，相当于where id=1 and name=xxx
• should（or），满足一个条件即可，相当于 where id=1 or name=xxx
• must_not（!=），所有条件必须满足不等于，相当于where id!=1 and name!=xxx

GET /user/_doc/_search
{
  "query":{
    "bool":{
      "must":[ #and条件
        {
          "match":{
            "user_name":"zero"
          }
        },
        {
          "match":{
            "age":18
          }
        }
      ]
    }
  }
}

7. 过滤查询

查询user_name带有zero，并筛选出18

GET /user/_doc/_search
{
  "query":{
    "bool":{
      "should":[
        {
          "match":{
            "user_name":"zero"
          }
        }
      ],
      "filter":{
        "range":{
          "age":{
            "gt":18,
            "lte":20
          }
        }
      }
    }
  }
}

8. 精准查询

term：通过倒排索引指定的词条进行精确的查找，配合keyword类型的字段，都不分词，直接匹配
match：查询之前会通过分词器解析，解析后再进行查询
两个类型：text（会被分词器解析）和keyword（不会被分词器解析）

#### term
GET /user/_doc/_search
{
  "query":{
    "bool":{
      "must":[
        {
          "term":{
            "user_name":"zero error"
          }
        }
      ]
    }
  }
}
#结果如下
#! [types removal] Specifying types in search requests is deprecated.
{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 0,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  }
}

#### must
GET /user/_doc/_search
{
  "query":{
    "bool":{
      "must":[
        {
          "match":{
            "user_name":"zero error"
          }
        }
      ]
    }
  }
}
## 执行结果
#! [types removal] Specifying types in search requests is deprecated.
{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 3,
      "relation" : "eq"
    },
    "max_score" : 1.3733702,
    "hits" : [
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "3",
        "_score" : 1.3733702,
        "_source" : {
          "user_name" : "zero error",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "1",
        "_score" : 0.41299206,
        "_source" : {
          "user_name" : "zero",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        }
      },
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "4",
        "_score" : 0.31387398,
        "_source" : {
          "user_name" : "zero two",
          "age" : 20,
          "create_at" : "2021-10-25 12:00:00"
        }
      }
    ]
  }
}

9. 高亮查询

GET /user/_doc/_search
{
  "query":{
    "match":{
      "user_name":"error"
    }
  },
  "highlight": {  #高亮配置
    "fields": {
      "user_name":{} 
    }
  }
}
#查询结果
#! [types removal] Specifying types in search requests is deprecated.
{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 1,
      "relation" : "eq"
    },
    "max_score" : 1.0594962,
    "hits" : [
      {
        "_index" : "user",
        "_type" : "_doc",
        "_id" : "3",
        "_score" : 1.0594962,
        "_source" : {
          "user_name" : "zero error",
          "age" : 18,
          "create_at" : "2021-10-26 12:00:00"
        },
        "highlight" : {  #高亮结果
          "user_name" : [
            "zero error"
          ]
        }
      }
    ]
  }
}