elasticsearch简明教程

安装配置

安装

• 官网下载www.elastic.co/downloads安装，这里下载elasticsearch-6.8.1.deb 。
• 同时下载ik分词器（elasticsearch-analysis-ik-6.8.1.zip）和kibana-6.8.1-amd64.deb.
• 注意分词器的版本,kibana的版本和elasticsearch的版本保持一致。

配置elasticsearch

• 修改elasticsearch配置文件 ubuntu在/etc/default/elasticsearch
  centos在/etc/sysconfig/elasticsearch，配置JAVA_HOME：

      JAVA_HOME=/opt/xxx/jdk

• 修改/etc/elasticsearch/jvm.options文件:

      #elasticsearch默认使用本机所有内存
      -Xms2g
      -Xmx2g

• 修改/etc/elasticsearch/elasticsearch.yml文件

      cluster.name: elmode # 集群名称
      node.name: node-1 #节点名称
      network.host: 0.0.0.0 #配置ip，4个0表示可被所有地址访问
      port:9200 #端口，默认9200
      discovery.zen.ping.unicast.hosts: ["test"] #主机名
      #禁用插件
      bootstrap.memory_lock: false
      bootstrap.system_call_filter: false

• 将分词器解压并复制到/usr/share/elasticsearch/plugins目录.
• 启动elasticsearch

  sudo systemctl start elasticsearch.service

• kibana配置(/etc/kibana.yml)

    server.host: "10.1.3.15" #配置ip
    elasticsearch.hosts: ["http://10.1.3.16:9200"] #配置要监控的elasticsearch节点

• 启动kibana

sudo systemctl start kibana.service

• 访问127.0.0.1:5601
  kibana启动较慢，如果页面出现 kibana service is not yet,可稍等一会儿。
  启动成功即可出现kibana页面，访问左侧菜单的DevTools选项，可在页面输入dsl语句
  直接操作elasticsearch.

DSL常用操作

索引操作

    #查询全部索引
    GET /_cat/indices

    #创建索引，test为索引名，并指定分片数和副本数
    PUT /test  
    {
	"settings": {
	    "number_of_shards": 5,
	    "number_of_replicas": 1
	}
    }

    #查询索引
    GET /test

    #删除索引
    DELETE /test

映射字段

    #创建映射字段，goods为type名，相当于数据库表
    PUT /test/_mapping/goods
    {
	"properties": {
	    "title": {
		"type": "text",
		"analyzer": "ik_max_word" 
	    },
	    "images": {
		"type": "keyword", 
		"index": false
	    },
	    "price": {
		"type": "long"
	    }
	}
    }

    #查询索引
    GET /test/_mapping

文档操作

    #增加文档，相当于增加一条数据库记录，test是索引名，goods是type名。
    #10001为手动指定的id,无指定可自动生成。
    POST /test/goods/10001
    {
	"title": "小米手机，为发烧而生",
	"image": "http://www.xxx.com/xxx.jpg",
	"price": 1999
    }

    #查询文档
    GET /test/_search

    #指定查询条件
    GET /test/_search
    {
	"query": {
	    "match": {
		"title": "小米"
	    }
	}
    }

    #修改文档，修改id为10001的价格
    POST /test/goods/10001/_update
    {
	"doc": {
	    "price": 2399.9
	}
    }

    #删除文档，指定lc9-为文档id
    DELETE /test/goods/lc9-dnQBt5qA1asqTRh7

查询

• 查询所有

GET /test/_search

• id查询

GET /test/_search/10001

• 匹配查询

    #匹配全部
    GET /test/_search
    {
	"query": {
	    "match_all": {}
	}
    }
    #匹配短句
    GET /test/_search
    {
	"query": {
	    "match_phrase": {
		"title": "华为手机"
	    }
	}
    }

• 其它查询

    #多字段匹配
    GET /test/_search
    {
	"query": {
	    "multi_match": {
		"query": "小米",
		"fields": ["title","image"]
	    }
	}
    }
    #词条查询，词条作为整体查询，不分词
    GET /test/_search
    {
	"query": {
	    "term": {
		"title": {
		    "value": "华为"
		}
	    }
	}
    }

    #多词条查询
    GET /test/_search
    {
	"query": {
	    "terms": {
		"title": [
		    "华为",
		    "手机"
		]
	    }
	}
    }

    #范围查询
    GET /test/_search
    {
	"query": {
	    "range": {
		"price": {
		    "gte": 1000,
		    "lte": 2000
		}
	    }
	}
    }
    #模糊查询，允许出现拼写误差，误差超过2个字符，仅限英文。
    GET /test/_search
    {
	"query": {
	    "fuzzy": {
		"title": {
		    "value": "oppe"
		}
	    }
	}
    }

• 布尔查询

#must 交集 should 并集
GET /test/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "price": {
              "gte": 1999,
              "lte": 2999
            }
          }
        },{
          "range": {
            "price": {
              "gte": 2999,
              "lte": 3999
            }
          }
        }
      ]
    }
  }
}

过滤

GET /test/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "match": {
            "title":  "手机"
          }
        }
      ],"filter": {
        "term": {
          "title": "oppo"
        }
      }
    }
  }
}

排序, 分页，高亮，结果集过滤

• 此处的过滤是需要显示哪些字段

GET /test/_search
{
  "query": {
   "match": {
     "title": "手机"
   }
  },"sort": [
    {
      "price": {
        "order": "asc"
      }
    },{
      "_id":{
        "order": "desc"
      }
    }
  ],
  "from": 0,
  "size": 5,
  "highlight": {
    "fields":{"title":{}},
    "pre_tags": "",
    "post_tags": ""
  },
  "_source": ["title","price"]
}

聚合.度量

GET /test/_search
{
  "size": 0, 
  "aggs": {
    "brand": {
      "terms": {
        "field": "attr.brand.keyword"
      },
      "aggs": {
        "avg_price": {
          "avg": {
            "field": "price"
          }
        }
      }
    }
  }
}

java操作elasticsearch

引入包

  <dependency>
    <groupId>org.springframework.bootgroupId>
    <artifactId>spring-boot-starter-data-elasticsearchartifactId>
  dependency>

定义pojo类，以及与文档的映射

  //实体类与文档的映射，分别指定了索引，类型，分片数，副本数。
  @Document(indexName = "user", type = "info", shards = 3, replicas = 3)
  public class User {

      //指定id
      @Id
      private Long id;

      //映射字段类型，并指定分词器
      @Field(type = FieldType.Text, analyzer = "ik_max_word")
      private String name;

      @Field(type = FieldType.Integer)
      private Integer age;

      @Field(type = FieldType.Keyword, index = false)
      private String password;
	
      public User() {}
	
      public User(Long id, String name, Integer age, String password) {
	  this.id = id;
	  this.name = name;
	  this.age = age;
	  this.password = password;
      }

      //省略getter,setter...

  }

索引和文档相关操作

• 文档的操作定义了UserRepository接口，接口定义如下：

public interface UserRepository extends ElasticsearchRepository<User, Long> {

}

  @SpringBootTest
  @RunWith(SpringRunner.class)
  public class ElasticSearchTest {

      @Autowired
      private ElasticsearchRestTemplate restTemplate;
	
      @Autowired
      private UserRepository userRepository;
	
      @Test
      public void elasticsearch() {
	  //创建索引
	  this.restTemplate.createIndex(User.class);
		
	  //创建映射
	  this.restTemplate.putMapping(User.class);
      }
	
      @Test
      public void repository() {
	  //新增文档（相当于新增表记录）
	  //this.userRepository.save(new User(10001L,"张天天",18,"123465"));
		
	  //批量新增
	  List<User> users = Arrays.asList(
					   new User(10002L,"赵重",21,"123456"),
					   new User(10003L,"钱多多",23,"123456"),
					   new User(10004L,"孙湘忆",22,"123456"),
					   new User(10005L,"李明明",18,"123456"),
					   new User(10006L,"周涛",20,"123456"),
					   new User(10007L,"吴元",35,"123456")
					   );
	  this.userRepository.saveAll(users);
      }
  }

查询

  public interface UserRepository extends ElasticsearchRepository<User, Long> {

      //自定义查询，方法名符合模板即可直接使用。
      List<User> findByAgeBetween(Integer age1, Integer age2);
      //官方文档查询的方法名模板示例：
      /##
       # 
       # findByName : name为字段名，按name查询，如按其它字段查询则可更改字段，如: findByAge.
       # fintByNameNot : 按name查询取反。
       # findByAgeLessThan: LessThan固定写法，查询比指定年龄小的结果
       # findByAgeGreateThan: 查询比指定年龄大的结果
       # findByAgeBetween: 查询年龄区间
       # findByNameAndAge: 多个字段查询
       # ....
       # 更多模板参考官方文档。
       # 
       #/
  }

高级查询

直接使用DSL语句

  //以下查询年龄在age1-age2，age3-age4的并集。
  //dsl语句中?0,?1,?2,?3是占位符，分别匹配方法中的4个参数
  //调用该方法时传入年龄参数即可
  @Query("{\n" + 
	 "    \"bool\": {\n" + 
	 "      \"should\": [\n" + 
	 "        {\n" + 
	 "          \"range\": {\n" + 
	 "            \"age\": {\n" + 
	 "              \"gte\": \"?0\",\n" + 
	 "              \"lte\": \"?1\"\n" + 
	 "            }\n" + 
	 "          }\n" + 
	 "        },{\n" + 
	 "          \"range\": {\n" + 
	 "            \"age\": {\n" + 
	 "              \"gte\": \"?2\",\n" + 
	 "              \"lte\": \"?3\"\n" + 
	 "            }\n" + 
	 "          }\n" + 
	 "        }\n" + 
	 "      ]\n" + 
	 "    }\n" + 
	 "  }")
  List<User> findByQuery(Integer age1, Integer age2, Integer age3, Integer age4);

构建查询器实现分页排序高亮

  @Test
  public void queryBuild() {
      NativeSearchQueryBuilder queryBuild  = new NativeSearchQueryBuilder();
      queryBuild.withQuery(QueryBuilders.matchQuery("name", "天天"));
		
      //分页
      queryBuild.withPageable(PageRequest.of(0, 2));
		
      //排序
      queryBuild.withSort(SortBuilders.fieldSort("age").order(SortOrder.ASC));
		
      //高亮
      queryBuild.withHighlightBuilder(new HighlightBuilder().field("name").preTags("").postTags(""));
      Page<User> userPage = this.userRepository.search(queryBuild.build());
      List<User> list = userPage.getContent();
      for (User user : list) {
	  System.out.println(user.getName() + " : " + user.getAge());
      }
  }

一个简单实例

模拟购物网站首页通过关键词搜索。

定义实体类到文档的映射

  @Document(indexName = "goods", type = "info", shards = 3, replicas = 2)
  public class Goods {

      @Id
      private Long skuId;

      @Field(type = FieldType.Keyword, index = false)
      private String pic;

      @Field(type = FieldType.Text, analyzer = "ik_max_word")
      private String title;

      @Field(type = FieldType.Keyword,  index = false)
      private BigDecimal price;

      @Field(type = FieldType.Long)
      private Long sale; // 销量

      @Field(type = FieldType.Date)
      private Date createTime;

      @Field(type = FieldType.Long)
      private Long brandId; // 品牌id

      @Field(type = FieldType.Keyword)
      private String brandName; // 品牌名称

      @Field(type = FieldType.Long)
      private Long categoryId; // 分类id

      @Field(type = FieldType.Keyword)
      private String categoryName; // 分类名称

      @Field(type = FieldType.Nested)
      private List<?> attrs; // 搜索属性

      //省略getter，setter，构造方法
  }

创建索引

  @Test
  public void init() {
      this.restTemplate.createIndex(Goods.class);
      this.restTemplate.putMapping(Goods.class);
  }

准备测试数据

  @Test
  public void data() {
      List<SearchAttr> sl = Arrays.asList(
					  new SearchAttr(10001L,"运行内存","8GB"),
					  new SearchAttr(10002L,"屏幕","ALOMD"),
					  new SearchAttr(10003L,"存储","128GB"));
      List<SearchAttr> sl2 = Arrays.asList(
					   new SearchAttr(10004L,"运行内存","6GB"),
					   new SearchAttr(10005L,"屏幕","LCD"),
					   new SearchAttr(10006L,"存储","256GB"));
      List<SearchAttr> sl3 = Arrays.asList(
					   new SearchAttr(10007L,"运行内存","4GB"),
					   new SearchAttr(10008L,"屏幕","ALOMD"),
					   new SearchAttr(10009L,"电池","4000mah"));
      List<SearchAttr> sl4 = Arrays.asList(
					   new SearchAttr(10010L,"硬盘","固态"),
					   new SearchAttr(10011L,"分辨率","1920x1080"),
					   new SearchAttr(10012L,"颜色","魅夜黑"));
      List<SearchAttr> sl5 = Arrays.asList(
					   new SearchAttr(10013L,"硬盘","固态"),
					   new SearchAttr(10014L,"分辨率","2560x1440"),
					   new SearchAttr(10015L,"颜色","晨雾白"));
	
		
      List<Goods> sd = Arrays.asList(
				     new Goods(20001L,"","小米10手机",4999.0,1000L,new Date(),30001L,"小米",40001L,"手机",sl),
				     new Goods(20002L,"","小米10pro手机",5999.0,1000L,new Date(),30001L,"小米",40001L,"手机",sl2),
				     new Goods(20003L,"","红米 note 8手机",1999.0,1000L,new Date(),30001L,"小米",40001L,"手机",sl3));
		
      List<Goods> sd2 = Arrays.asList(
				      new Goods(20004L,"","华为p40手机",4999.0,800L,new Date(),30002L,"华为",40001L,"手机",sl),
				      new Goods(20005L,"","华为nova30手机",2999.0,700L,new Date(),30002L,"华为",40001L,"手机",sl2),
				      new Goods(20006L,"","荣耀play手机",1999.0,1500L,new Date(),30002L,"华为",40001L,"手机",sl3));
		
      List<Goods> sd3 = Arrays.asList(
				      new Goods(20007L,"","联想Air笔记本",5999.0,600L,new Date(),30003L,"联想",40002L,"笔记本",sl5),
				      new Goods(20008L,"","联想小新笔记本",4888.0,500L,new Date(),30003L,"联想",40002L,"笔记本",sl4));
		
      this.goodsRepository.saveAll(sd);
      this.goodsRepository.saveAll(sd2);
      this.goodsRepository.saveAll(sd3);
  }

编写DSL

该查询通过##手机##关键字查询小米和华为8GB运存的手机。

GET /goods/_search
{
  "query": {
    "bool": {
      "must": [
        {"match": {
          "title": {
            "query": "手机",  #查询关键词
            "operator": "and"
          }
        }}
      ],
      "filter": [
        {
        "terms":{
          "brandId": [30001,30002] #过滤条件（小米品牌和华为品牌）
        }
      },
      {
        "terms":{
          "categoryId": [40001] # 过滤条件(分类：手机)
        }
      },
      {
        "bool":{
          "must":[
            {
              "nested":{ #嵌套属性关键字
                "path": "attrs",
                "query":{
                  "bool":{
                    "must":[
                      {
                        "term":{
                          "attrs.attrId": 10001 #搜索属性（8GB运存）
                        }
                      }
                      ]
                  }
                }
              }
            }
            ]
        }
      }
      ]
    }
  }
}

构建查询

  public class GoodsTest {

      @Autowired
      private RestHighLevelClient client;
	
      @Test
      public void  buildQueryDsl() throws IOException {
	  //查询条件构建器
	  SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
		
	  BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
		
	  //构建查询条件
	  boolQueryBuilder.must(QueryBuilders.matchQuery("title", "手机").operator(Operator.AND));
		
	  //构建过滤条件(品牌)
	  String[] brand = new String[] {"30001","30002"};
	  boolQueryBuilder.filter(QueryBuilders.termsQuery("brandId", brand));
		
	  //构建过滤条件（分类）
	  String[] category = new String[] {"40001"};
	  boolQueryBuilder.filter(QueryBuilders.termsQuery("categoryId", category));
		
	  //构建规格属性查询条件
	  String attrId = "10001";
	  BoolQueryBuilder subQuery = QueryBuilders.boolQuery();
	  subQuery.must(QueryBuilders.termQuery("attrs.attrId", attrId));
		
	  BoolQueryBuilder attrsQuery = QueryBuilders.boolQuery();
	  attrsQuery.must(QueryBuilders.nestedQuery("attrs", subQuery, ScoreMode.None));
	  boolQueryBuilder.filter(attrsQuery);
		
	  sourceBuilder.query(boolQueryBuilder);
	  //构建分页
	  sourceBuilder.from(0);
	  sourceBuilder.size(5);
		
	  //构建排序
	  sourceBuilder.sort("price", SortOrder.ASC);
		
	  //构建高亮
	  sourceBuilder.highlighter(new HighlightBuilder().field("title").preTags("").postTags(""));
		
	  //构建聚合(根据品牌聚合)
	  sourceBuilder.aggregation(AggregationBuilders.terms("brnadIdAdd").field("brandId"));
		
	  SearchRequest searchRequest = new SearchRequest("goods"); //要查询的索引
	  searchRequest.types("info");
	  searchRequest.source(sourceBuilder);
	  SearchResponse resp = client.search(searchRequest, RequestOptions.DEFAULT);
		
	  System.out.println(resp);
      }
  }

解析结果集

返回的结果集为json,常规的方法解析即可。

  {
      "took": 20,
      "timed_out": false,
      "_shards": {
	  "total": 3,
	  "successful": 3,
	  "skipped": 0,
	  "failed": 0
      },
      "hits": {
	  "total": 2,
	  "max_score": null,
	  "hits": [{
	      "_index": "goods",
	      "_type": "info",
	      "_id": "20004",
	      "_score": null,
	      "_source": {
		  "skuId": 20004,
		  "pic": "",
		  "title": "华为p40手机",
		  "price": 4999.0,
		  "sale": 800,
		  "createTime": 1599991429799,
		  "brandId": 30002,
		  "brandName": "华为",
		  "categoryId": 40001,
		  "categoryName": "手机",
		  "attrs": [{
		      "attrId": 10001,
		      "attrName": "运行内存",
		      "attrValue": "8GB"
		  }, {
		      "attrId": 10002,
		      "attrName": "屏幕",
		      "attrValue": "ALOMD"
		  }, {
		      "attrId": 10003,
		      "attrName": "存储",
		      "attrValue": "128GB"
		  }]
	      },
	      "highlight": {
		  "title": ["华为p40手机"]
	      },
	      "sort": ["4999.0"]
	  }, {
	      "_index": "goods",
	      "_type": "info",
	      "_id": "20001",
	      "_score": null,
	      "_source": {
		  "skuId": 20001,
		  "pic": "",
		  "title": "小米10手机",
		  "price": 4999.0,
		  "sale": 1000,
		  "createTime": 1599991429799,
		  "brandId": 30001,
		  "brandName": "小米",
		  "categoryId": 40001,
		  "categoryName": "手机",
		  "attrs": [{
		      "attrId": 10001,
		      "attrName": "运行内存",
		      "attrValue": "8GB"
		  }, {
		      "attrId": 10002,
		      "attrName": "屏幕",
		      "attrValue": "ALOMD"
		  }, {
		      "attrId": 10003,
		      "attrName": "存储",
		      "attrValue": "128GB"
		  }]
	      },
	      "highlight": {
		  "title": ["小米10手机"]
	      },
	      "sort": ["4999.0"]
	  }]
      },
      "aggregations": {
	  "lterms#brnadIdAdd": {
	      "doc_count_error_upper_bound": 0,
	      "sum_other_doc_count": 0,
	      "buckets": [{
		  "key": 30001,
		  "doc_count": 1
	      }, {
		  "key": 30002,
		  "doc_count": 1
	      }]
	  }
      }
  }