elasticsearch7.1.1学习之整合springboot——data整合
9.30的时候maven公库里发布了spring-data-elasticsearch3.2.0的正式版本，那么主要新的特性是支持响应式编程（这个响应式编程通过异步返回的方式来提高吞吐量，可以和springwebflux一起使用，主要返回的对象有mono,flux）和升级到支持elasticsearch6.8.1，而对应的springboot版本是2.2.0。下图是版本对应的关系
| Spring Data Release Train | Spring Data Elasticsearch | Elasticsearch | Spring Boot |
|---|---|---|---|
| Moore[1] |
3.2.x[1] |
6.8.1 / 7.x[2] |
2.2.0[1] |
| Lovelace |
3.1.x |
6.2.2 / 7.x[2] |
2.1.x |
| Kay[3] |
3.0.x[3] |
5.5.0 |
2.0.x[3] |
| Ingalls[3] |
2.1.x[3] |
2.4.0 |
1.5.x[3] |
话不多说，开始编写代码,这里用的是springboot2.2.0版本配套的es starter
4.0.0
org.springframework.boot
spring-boot-starter-parent
2.2.0
com.gdut.imis
es-data-demo
0.0.1-SNAPSHOT
es-data-demo
Demo project for Spring Boot
1.8
org.springframework.boot
spring-boot-starter-data-elasticsearch
org.springframework.boot
spring-boot-starter-webflux
org.projectlombok
lombok
true
org.springframework.boot
spring-boot-starter-test
test
org.junit.vintage
junit-vintage-engine
org.springframework.boot
spring-boot-maven-plugin
写配置：
由于官方建议用高版本的客户端，所以使用restHighLevelClient
package com.gdut.imis.esdatademo.configuration;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.core.convert.support.DefaultConversionService;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
import org.springframework.data.elasticsearch.client.reactive.ReactiveRestClients;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
import org.springframework.data.elasticsearch.core.ElasticsearchEntityMapper;
import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions;
import org.springframework.data.geo.Point;
import java.time.Duration;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
/**
* @author lulu
* @Date 2019/10/7 14:09
*/
@Configuration
public class EsConfiguration extends AbstractElasticsearchConfiguration {
@Bean
@Override
public RestHighLevelClient elasticsearchClient() {
//链接配置
ClientConfiguration clientConfiguration = ClientConfiguration.builder().connectedTo("localhost:9200")
.withConnectTimeout(Duration.ofSeconds(15)).withSocketTimeout(Duration.ofSeconds(15))
//.withBasicAuth()
.build();
return RestClients.create(clientConfiguration).rest();
}
@Bean
//reactive配置
ReactiveElasticsearchClient client() {
ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo("localhost:9200")
.build();
return ReactiveRestClients.create(clientConfiguration);
}
@Bean
@Override
public EntityMapper entityMapper() {
//entityMapper可以自定义怎么把实体对象和json进行映射
// elasticsearchMappingContext返回一个具有@Document注解的实体类集合上下文
//DefaultConversionService里定义一系列的转换方式
ElasticsearchEntityMapper entityMapper = new ElasticsearchEntityMapper(
elasticsearchMappingContext(), new DefaultConversionService()
);
//conversionService用于转换对象,如果没有自定义的显式配置，则有默认实现
entityMapper.setConversions(elasticsearchCustomConversions());
return entityMapper;
}
@Bean
@Override
//自定义的转换
public ElasticsearchCustomConversions elasticsearchCustomConversions() {
return new ElasticsearchCustomConversions(
Arrays.asList(new PointToMap(), new MapToPoint()));
}
@WritingConverter
static class PointToMap implements Converter> {
@Override
public Map convert(Point p) {
Map map = new HashMap();
map.put("user_lat", p.getX());
map.put("user_lon", p.getY());
return map;
}
}
@ReadingConverter
static class MapToPoint implements Converter, Point> {
@Override
public Point convert(Map map) {
return new Point(map.get("user_lat"), map.get("user_lon"));
}
}
}
定义了一个User实体，这里面的注解类型有
-
@Id：作用在字段上面，用于标识对象，类似主键的作用。 -
@Document：作用在类上面，表明该类是映射到数据库的对象（实体类）。其中比较重要的属性是：-
indexName：用于存储此实体的索引的名称 -
type：映射类型。如果未设置，则使用小写的类的简单名称，最好设置为"_doc"，这样不会有warn的日志。 -
shards：索引的分片数。 -
replicas：索引的副本数。 -
refreshIntervall：索引的刷新间隔。用于索引创建。默认值为“ 1s”。 -
indexStoreType：索引的索引存储类型。用于索引创建。默认值为“ fs”。 -
createIndex：配置是否在存储库引导中创建索引。默认值为true。 -
versionType：版本管理的配置。默认值为EXTERNAL。
-
-
@Transient：默认情况下，所有私有字段都映射到文档，此注释作用的字段将不会映射到数据库中 -
@Field：在字段级别应用并定义字段的属性，大多数属性映射到各自的Elasticsearch映射定义：-
name：字段名称，将在Elasticsearch文档中表示，如果未设置，则使用Java字段名称。 -
type：字段类型，可以是Text，Integer，Long，Date，Float，Double，Boolean，Object，Auto，Nested，Ip，Attachment，Keyword之一。 -
format和日期类型的pattern自定义定义。 -
store：标记是否将原始字段值存储在Elasticsearch中，默认值为false。 -
analyzer，searchAnalyzer，normalizer用于指定自定义自定义分析和正规化。 -
copy_to：将多个文档字段复制到的目标字段。 -
fielddata: 作用于text类型，设置为true的字段可以对其进行聚合
-
-
@GeoPoint：将字段标记为geo_point数据类型。如果字段是GeoPoint类的实例，则可以省略。
package com.gdut.imis.esdatademo.entity;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Transient;
import org.springframework.data.elasticsearch.annotations.*;
import java.util.List;
/**
* @author lulu
* @Date 2019/10/7 14:19
*/
@Document(indexName = "user_info",shards = 2,type = "_doc")
@Data
public class User {
@Id
private Integer userId;
@Field(name="user_name",type = FieldType.Keyword)
private String name;
private Address location;
@Field(name="user_birthday",type=FieldType.Date,format = DateFormat.date_hour_minute_second )
private String birthDay;
@Transient
private List jobList;
}
package com.gdut.imis.esdatademo.repository;
import com.gdut.imis.esdatademo.entity.User;
import org.springframework.data.domain.Page;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* @author lulu
* @Date 2019/10/7 14:51
*/
@Repository
public interface UserRepository extends ElasticsearchRepository {
/**
* 根据用户名模糊查询
* @param name
* @return
*/
List findAllByNameLike(String name);
/**
* 按用户名模糊查询and根据Address属性下的city进行查询，并且按照id排序
* @param userName
* @param city
* @return
*/
List findUsersByNameLikeAndLocationCityEqualsOrderByUserId(
String userName,
String city
);
}
定义好实体以后，再继承ElasticsearchRepository就可以了，其中可以自定义一些方法名，orm框架会根据一定方法把他解析成对应的查询语句并且执行，这里定义方法可能会有一个点要注意，就是如果User对象有一个localtionCity属性，他的Address属性有一个city属性，此时需要把LocationCity改为Location_City才可以查找得到,其实用法都是差不多，主要是跟据条件查询
另外的一个job类
package com.gdut.imis.esdatademo.entity;
import lombok.Data;
import lombok.experimental.Accessors;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
/**
* @author lulu
* @Date 2019/10/7 19:37
*/
@Document(indexName = "user_job",replicas = 2,shards = 1, type = "_doc",createIndex = false)
@Data
@Accessors(chain = true)
public class Job {
private String desc;
@Id
private String jobName;
private Double salary;
}
package com.gdut.imis.esdatademo.repository;
import com.gdut.imis.esdatademo.entity.Job;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;
/**
* @author lulu
* @Date 2019/10/7 19:49
*/
@Repository
public interface JobRepository extends ElasticsearchRepository {
Page findByDescLike(String job, Pageable pageable);
}
这里面主要是一个分页的方法，其实用法比较普遍和简单，也没什么好说的，如果想自定义查询，也可以使用ElasticsearchRestTemplate作为查询工具，template的方法还是很多的，而且构造的查询也可以相对复杂，并且支持聚合、批量操作等等。配置和上面一样，直接注入就可以使用,下面这个则是支持reactive流式
package com.gdut.imis.esdatademo.repository;
import com.gdut.imis.esdatademo.entity.Job;
import org.springframework.data.elasticsearch.repository.ReactiveElasticsearchRepository;
/**
* @author lulu
* @Date 2019/10/9 13:23
*/
@Repository
public interface JobReactiveRepository extends ReactiveElasticsearchRepository {
}
例子
package com.gdut.imis.esdatademo.service;
import com.gdut.imis.esdatademo.entity.Job;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.ParsedCardinality;
import org.elasticsearch.search.aggregations.metrics.stats.ParsedStats;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.ReactiveElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @author lulu
* @Date 2019/10/7 21:26
*/
@Service
public class JobService {
@Autowired
private ElasticsearchRestTemplate template;
@Autowired
private ReactiveElasticsearchTemplate reactiveTemplate;
public Flux queryDemo(String jobName,Double from,Double to){
/**
* 构造布尔查询
*/
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
//模糊查询名字
QueryStringQueryBuilder name = QueryBuilders.queryStringQuery(jobName).field("jobName");
boolQuery.must(name);
//限定薪水范围
RangeQueryBuilder salary = QueryBuilders.rangeQuery("salary").lte(to).gte(from);
boolQuery.must(salary);
NativeSearchQuery query=new NativeSearchQuery(boolQuery);
//定义排序
Sort.TypedSort sort = Sort.sort(Job.class);
Sort descending = sort.by(Job::getSalary).descending();
//分页
PageRequest request=PageRequest.of(0,10,descending);
query.setPageable(request);
Flux jobFlux = reactiveTemplate.find(query, Job.class);
return jobFlux;
}
public Map termDemo(){
MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
//terms聚合，会以一个个桶的形式返回
TermsAggregationBuilder userCity = AggregationBuilders.terms("user_city").field("location.city");
NativeSearchQuery searchQuery=new NativeSearchQuery(queryBuilder);
searchQuery.addIndices("user_info");
searchQuery.setSearchType(SearchType.DEFAULT);
searchQuery.addTypes("_doc");
searchQuery.addAggregation(userCity);
ParsedStringTerms city = template.query(searchQuery, response -> (ParsedStringTerms) response.getAggregations().getAsMap().get("user_city"));
Map map= city.getBuckets().stream().filter(e->((Bucket) e).getDocCount()>5).collect(Collectors.toMap(bucket -> bucket.getKey().toString(), Bucket::getDocCount));
return map;
}
public Map aggDemo(){
MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
StatsAggregationBuilder agg = AggregationBuilders.stats("salary_stats").field("salary");
CardinalityAggregationBuilder agg2 = AggregationBuilders.cardinality("user_address_code_stats").field("location.code");
NativeSearchQuery searchQuery=new NativeSearchQuery(queryBuilder);
searchQuery.addIndices("user_job","user_info");
searchQuery.addTypes("_doc");
searchQuery.addAggregation(agg);
searchQuery.addAggregation(agg2);
Map query = template.query(searchQuery, response -> response.getAggregations().asMap());
ParsedStats stats= (ParsedStats) query.get("salary_stats");
ParsedCardinality cardinality= (ParsedCardinality) query.get("user_address_code_stats");
return query;
}
}
controller，其实webflux还有另一种编程方式，是handler+routing的方式开发，有兴趣的可以自行了解下，因为我对这个webflux只停留于简单了解的状态，就不献丑了哈哈，以后有机会再补上
package com.gdut.imis.esdatademo.controller;
import com.gdut.imis.esdatademo.entity.Address;
import com.gdut.imis.esdatademo.entity.Job;
import com.gdut.imis.esdatademo.entity.User;
import com.gdut.imis.esdatademo.repository.JobReactiveRepository;
import com.gdut.imis.esdatademo.repository.JobRepository;
import com.gdut.imis.esdatademo.repository.UserRepository;
import com.gdut.imis.esdatademo.service.JobService;
import org.elasticsearch.search.aggregations.Aggregation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.geo.Point;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Flux;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* @author lulu
* @Date 2019/10/7 14:54
*/
@RestController
public class TestController {
@Autowired
private UserRepository userRepository;
@Autowired
private JobRepository jobRepository;
@Autowired
private JobReactiveRepository jobReactiveRepository;
@Autowired
private JobService jobService;
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
@GetMapping("/getUserByName")
public List userList(@RequestParam("name") String name) {
return userRepository.findAllByNameLike(name);
}
@GetMapping("/getUserByCity")
public List getUserList(@RequestParam("name")String name,
@RequestParam("city")String city){
return userRepository.findUsersByNameLikeAndLocationCityEqualsOrderByUserId(name,city);
}
@GetMapping("/getFlux")
public Flux queryDemo(@RequestParam("jobName") String jobName,@RequestParam("from")Double from,@RequestParam("to") Double to){
return jobService.queryDemo(jobName,from,to);
}
@GetMapping("/createUser")
public Iterable createUser(@RequestParam("from")Integer from,@RequestParam("to")Integer to) {
String[] country = {"uk", "china", "japan"};
String[] city = {"london", "gz", "tokyo"};
String[] street = {"a", "b", "c"};
List users = IntStream.rangeClosed(from, to).mapToObj(e -> {
User u = new User();
Point point = new Point(Math.random() * 100, Math.random() * 100);
u.setName("No" + e);
u.setUserId(e);
Address address = new Address();
address.setCountry(country[e % country.length]);
address.setCity(city[e % city.length]);
address.setStreet(street[e % street.length]);
address.setCode(e%2);
address.setPoint(point);
u.setLocation(address);
u.setBirthDay(dateFormat.format(new Date()));
return u;
}
).collect(Collectors.toList());
return userRepository.saveAll(users);
}
@GetMapping("/createManyJob")
public Iterable jobList(@RequestParam("from")Integer from,@RequestParam("to")Integer to) {
List jobList = IntStream.rangeClosed(from, to).mapToObj(e -> {
String res = UUID.randomUUID().toString();
Job job = new Job().setJobName(e + "")
.setDesc(res).setSalary(Math.random() * 1000);
return job;
}).collect(Collectors.toList());
// jobReactiveRepository.saveAll(jobList);
return jobRepository.saveAll(jobList);
}
@GetMapping("/getJobAgg")
public Map getJobAgg(){
return jobService.aggDemo();
}
@GetMapping("/getJob")
public List getJob(@RequestParam("name") String name,
@RequestParam("index") Integer index,
@RequestParam("size") Integer size
) {
Sort.TypedSort sort = Sort.sort(Job.class);
Sort descending = sort.by(Job::getSalary).descending();
PageRequest request = PageRequest.of(index, size, descending);
Page byDescLike = jobRepository.findByDescLike(name, request);
return byDescLike.getContent();
}
}
总结：这里面主要还是理解好es本身的json查询语句和聚合是怎么用的，再根据实际要求对应官方文档进行理解使用，其实版本升级以后就是某些api用法不同了，大体还是差不多的，高级的聚合查询我还不会，可以参考下面的链接，还有一个小配置，如果想看到template发送了什么返回了什么，可以配置日志级别
logging:
level:
org.springframework.data.elasticsearch.client.WIRE: trace参考链接：
template使用：https://blog.csdn.net/Topdandan/article/details/81436141
官方文档：https://docs.spring.io/spring-data/elasticsearch/docs/3.2.0.RELEASE/reference/html/#new-features
webflux:https://www.cnblogs.com/limuma/p/9315343.html,只是把文中的map改为和elasticsearch交互