前言: 由于实际项目中，需要操作多个elasticsearch库，于是便试了下集成到spring boot中。
一、前期准备
1.maven：
1.1 spring boot
org.springframework.boot spring-boot-dependencies 2.0.3.RELEASE
1.2 es client
6.3.2
org.elasticsearch elasticsearch ${es.version} org.elasticsearch.client elasticsearch-rest-client ${es.version} org.elasticsearch.client elasticsearch-rest-high-level-client ${es.version}
二、直接上代码
1.SysConfig类型
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
/**
* @author laimailai
* @version SysConfig.java, v 0.1 2018-07-20 16:27
*/
@Configuration
public class SysConfig {
/*********************es 配置*******************/
//主ES库
@Value("${spring.data.elasticsearch.host}")
private String esHost;
@Value("${spring.data.elasticsearch.port}")
private Integer esPort;
@Value("${spring.data.elasticsearch.username}")
private String esUsername;
@Value("${spring.data.elasticsearch.password}")
private String esPassword;
//业务ES库
@Value("${spring.data.business.elasticsearch.host}")
private String businessHost;
@Value("${spring.data.business.elasticsearch.port}")
private Integer businessPort;
@Value("${spring.data.business.elasticsearch.username}")
private String businessUsername;
@Value("${spring.data.business.elasticsearch.password}")
private String businessPssword;
// 省略get、set
2.SpringEsConfig类
import xx.SysConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author laimailai
* @version SpringEsConfig.java, v 0.1 2019-04-03 12:41
*/
@Configuration
public class SpringEsConfig {
@Autowired
private SysConfig sysConfig;
/**
* 主ES库
*
* @return
*/
@Bean(name = "elasticSearchClient", initMethod = "init", destroyMethod = "close")
public ElasticSearchClient elasticSearchClient() {
ElasticSearchClient elasticSearchClient = new ElasticSearchClient();
elasticSearchClient.setHost(sysConfig.getEsHost());
elasticSearchClient.setPort(sysConfig.getEsPort());
elasticSearchClient.setUsername(sysConfig.getEsUsername());
elasticSearchClient.setPassword(sysConfig.getEsPassword());
return elasticSearchClient;
}
/**
* 业务ES库
*
* @return
*/
@Bean(name = "elasticSearchBusinessClient", initMethod = "init", destroyMethod = "close")
public ElasticSearchClient elasticSearchBusinessClient() {
ElasticSearchClient elasticSearchClient = new ElasticSearchClient();
elasticSearchClient.setHost(sysConfig.getBusinessHost());
elasticSearchClient.setPort(sysConfig.getBusinessPort());
elasticSearchClient.setUsername(sysConfig.getBusinessUsername());
elasticSearchClient.setPassword(sysConfig.getBusinessPssword());
return elasticSearchClient;
}
/**
* 主ES库批量写入器
*
* @return
*/
@Bean(name = "elasticSearchUtil", initMethod = "init", destroyMethod = "destroy")
public ElasticSearchUtil elasticSearchUtil() {
ElasticSearchUtil elasticSearchUtil = new ElasticSearchUtil();
elasticSearchUtil.setElasticSearchClient(elasticSearchClient());
return elasticSearchUtil;
}
}
3.ElasticSearchClient类
import cn.hutool.core.bean.BeanUtil;
import com.xx.utils.NullUtil;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author laimailai
* @version ElasticSearchClient.java, v 0.1 2019-04-03 12:27
*/
public class ElasticSearchClient {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchClient.class);
/**
* 搜索上下文的时间,用来支持该批次
*/
private static final Long SCROLL_ALIVE_TIME = 5L;
private String host;
private int port;
private String username;
private String password;
private RestHighLevelClient restHighLevelClient;
public ElasticSearchClient() {
}
/**
* 初始化连接
*/
public void init() {
if (restHighLevelClient == null) {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
});
restHighLevelClient = new RestHighLevelClient(builder);
} else {
LOGGER.warn("ElasticSearch client 重复连接,host={},port={}", host, port);
}
}
/**
* 关闭连接
*/
public void close() {
try {
LOGGER.info("Closing elasticSearch client");
if (restHighLevelClient != null) {
restHighLevelClient.close();
}
} catch (final Exception e) {
LOGGER.error("Error closing ElasticSearch client: ", e);
}
}
/**
* Setter method for property host.
*
* @param host value to be assigned to property host
*/
public void setHost(String host) {
this.host = host;
}
/**
* Setter method for property port.
*
* @param port value to be assigned to property port
*/
public void setPort(int port) {
this.port = port;
}
/**
* Setter method for property username.
*
* @param username value to be assigned to property username
*/
public void setUsername(String username) {
this.username = username;
}
/**
* Setter method for property password.
*
* @param password value to be assigned to property password
*/
public void setPassword(String password) {
this.password = password;
}
/**
* Getter method for property restHighLevelClient.
*
* @return property value of restHighLevelClient
*/
public RestHighLevelClient getRestHighLevelClient() {
return restHighLevelClient;
}
/**********************ES通用方法**************************/
/**
* 根据id查询
*
* @param index
* @param type
* @param id
* @return
*/
public GetResponse getDocument(String index, String type, String id) {
GetRequest getRequest = new GetRequest(
index,
type,
id);
GetResponse response = null;
try {
response = restHighLevelClient.get(getRequest);
} catch (IOException e) {
LOGGER.error("getDocument ->> 根据id查询失败！index = {},type = {},id = {}", index, type, id, e);
}
return response;
}
/**
* 保存
*
* @param jsonMap
* @param index
* @param type
* @param id
* @return
*/
public IndexResponse saveDocument(Map jsonMap, String index, String type, String id) {
IndexRequest indexRequest = new IndexRequest(index, type, id)
.source(jsonMap);
IndexResponse response = null;
try {
response = restHighLevelClient.index(indexRequest);
} catch (IOException e) {
LOGGER.error("saveDocument ->> 保存失败！obj = {},index = {},type = {},id = {}",
jsonMap, index, type, id, e);
}
return response;
}
/**
* 修改
*
* @param jsonMap
* @param index
* @param type
* @param id
* @return
*/
public UpdateResponse updateDocument(Map jsonMap, String index, String type, String id) {
UpdateRequest request = new UpdateRequest(index, index, id)
.doc(jsonMap);
UpdateResponse response = null;
try {
response = restHighLevelClient.update(request);
} catch (IOException e) {
LOGGER.error("updateDocument ->> 修改失败！str = {},index = {},type = {},id = {}",
jsonMap, index, type, id, e);
}
return response;
}
/**
* 部分修改
*
* @param jsonMap
* @param index
* @param type
* @param id
* @return
*/
public UpdateResponse upsertDocument(Map jsonMap, String index, String type, String id) {
UpdateRequest request = new UpdateRequest(index, index, id).doc(jsonMap);
request.upsert(jsonMap, XContentType.JSON);
UpdateResponse response = null;
try {
response = restHighLevelClient.update(request);
} catch (IOException e) {
LOGGER.error("upsertDocument ->> 修改失败！str = {},index = {},type = {},id = {}",
jsonMap, index, type, id, e);
}
return response;
}
/**
* 批量插入
*
* @param entityList
* @param index
* @param type
* @param isAsync 是否异步
* @param
* @return
*/
public BulkResponse saveEntityBulk(List entityList, String index, String type, Boolean isAsync) {
BulkRequest request = new BulkRequest();
BulkResponse bulkResponse = null;
for (T t : entityList) {
Map jsonMap = BeanUtil.beanToMap(t, true, true);
IndexRequest indexRequest = new IndexRequest(index, type, String.valueOf(jsonMap.get("id")))
.source(jsonMap);
request.add(indexRequest);
}
request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
if (isAsync) {
saveBulkAsync(request, index, type, "saveEntityBulk");
} else {
saveBulkSync(request, index, type, "saveEntityBulk");
}
return bulkResponse;
}
/**
* 批量更新
*
* @param entityList
* @param index
* @param type
* @param isAsync 是否异步
* @param
* @return
*/
public BulkResponse updateEntityBulk(List entityList, String index, String type, Boolean isAsync) {
BulkRequest request = new BulkRequest();
BulkResponse response = null;
UpdateRequest updateRequest;
for (T t : entityList) {
Map jsonMap = BeanUtil.beanToMap(t, true, true);
updateRequest = new UpdateRequest(index, type, String.valueOf(jsonMap.get("id")))
.doc(jsonMap);
request.add(updateRequest);
}
if (isAsync) {
saveBulkAsync(request, index, type, "updateEntityBulk");
} else {
saveBulkSync(request, index, type, "updateEntityBulk");
}
return response;
}
/**
* 批量bulk(同步)
*
* @param request
* @param index
* @param type
* @param method
* @return
*/
private BulkResponse saveBulkSync(BulkRequest request, String index, String type, String method) {
BulkResponse bulkResponse = null;
try {
bulkResponse = restHighLevelClient.bulk(request);
if (bulkResponse.hasFailures()) {
LOGGER.error("saveBulkSync ->> 批量保存部分失败！index = {},type = {},errorMsg = {}",
index, type, bulkResponse.buildFailureMessage());
}
BulkItemResponse[] responses = bulkResponse.getItems();
printFailedItems(responses, method);
} catch (IOException e) {
LOGGER.error("saveBulkSync ->> 批量保存失败！index = {},type = {}",
index, type, e);
}
return bulkResponse;
}
/**
* 批量bulk(异步)
*
* @param request
* @param index
* @param type
* @param method
*/
private void saveBulkAsync(BulkRequest request, String index, String type, String method) {
restHighLevelClient.bulkAsync(request, new ActionListener() {
@Override
public void onResponse(BulkResponse bulkResponse) {
if (bulkResponse.hasFailures()) {
LOGGER.error("saveEntityBulkAsync ->> 异步批量保存部分失败！index = {},type = {},errorMsg = {}",
index, type, bulkResponse.buildFailureMessage());
}
BulkItemResponse[] responses = bulkResponse.getItems();
printFailedItems(responses, method);
}
@Override
public void onFailure(Exception e) {
LOGGER.error("saveEntityBulkAsync ->> 异步批量保存执行失败！index = {},type = {}",
index, type, e);
}
});
}
/**
* Scroll分页自定义搜索
*
* @param boolQueryBuilder
* @param index
* @param type
* @param size
* @param sortName
* @return
*/
public SearchResponse searchByScroll(BoolQueryBuilder boolQueryBuilder, String index, String type, Integer size, String sortName) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.size(size);
if (NullUtil.isNotNull(sortName)) {
searchSourceBuilder.sort(sortName, SortOrder.DESC);
}
searchRequest.source(searchSourceBuilder);
searchRequest.scroll(TimeValue.timeValueMinutes(SCROLL_ALIVE_TIME));
SearchResponse searchResponse = null;
try {
searchResponse = restHighLevelClient.search(searchRequest);
} catch (IOException e) {
LOGGER.error("searchByScroll ->> scroll分页搜索失败！index = {},type = {}",
index, type, e);
}
return searchResponse;
}
/**
* Scroll分页根据scrollId搜索
*
* @param index
* @param type
* @param scrollId
* @return
*/
public SearchResponse searchByScrollId(String index, String type, String scrollId) {
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(SCROLL_ALIVE_TIME));
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
SearchResponse searchResponse = null;
try {
searchResponse = restHighLevelClient.searchScroll(scrollRequest);
} catch (IOException e) {
LOGGER.error("searchByScrollId ->> scroll分页搜索失败！index = {},type = {}",
index, type, e);
}
return searchResponse;
}
/**
* 删除scrollId
*
* @param scrollId
*/
public void clearScrollId(String scrollId) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
try {
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest);
boolean succeeded = clearScrollResponse.isSucceeded();
LOGGER.info("clearScrollId ->> result = {}", succeeded);
} catch (IOException e) {
LOGGER.error("clearScrollId ->> 删除scrollId失败！scrollId = {}",
scrollId, e);
}
}
/**
* 根据多个条件搜索
*
* @param boolQueryBuilder
* @param index
* @param type
* @param page
* @param pageSize
* @param sortName
* @return
*/
public SearchResponse searchByQueryBuilder(BoolQueryBuilder boolQueryBuilder, String index, String type, Integer page, Integer pageSize, String sortName) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.from(page);
searchSourceBuilder.size(pageSize);
if (NullUtil.isNotNull(sortName)) {
searchSourceBuilder.sort(sortName, SortOrder.DESC);
}
searchRequest.source(searchSourceBuilder);
SearchResponse response = null;
try {
response = restHighLevelClient.search(searchRequest);
} catch (IOException e) {
LOGGER.error("searchByQueryBuilder ->> 根据多个条件搜索失败！index = {},boolQueryBuilder = {}",
index, boolQueryBuilder.toString(), e);
}
return response;
}
/**
* 根据多个条件搜索聚合
*
* @param boolQueryBuilder
* @param index
* @param type
* @param aggregationBuilder
* @return
*/
public SearchResponse aggByQueryBuilder(BoolQueryBuilder boolQueryBuilder, String index, String type, AggregationBuilder aggregationBuilder) {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.from(0);
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(aggregationBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = null;
try {
response = restHighLevelClient.search(searchRequest);
} catch (IOException e) {
LOGGER.error("searchByQueryBuilder ->> 根据多个条件搜索失败！index = {},boolQueryBuilder = {}",
index, boolQueryBuilder.toString(), e);
}
return response;
}
/**
* 打印错误id
*
* @param responses
* @param method
* @param
*/
public void printFailedItems(BulkItemResponse[] responses, String method) {
if (null == responses || responses.length == 0) {
return;
}
List ids = new ArrayList<>();
for (BulkItemResponse bulkItemResponse : responses) {
if (bulkItemResponse.isFailed()) {
BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
ids.add(failure.getId());
}
}
if (ids.size() > 0) {
LOGGER.error(method + " ->> 部分保存失败！ids = {}", ids.toString());
}
}
}
4.ElasticSearchUtil类
import org.elasticsearch.action.bulk.BackoffPolicy;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
/**
* @author laimailai
* @version ElasticSearchUtil.java, v 0.1 2018-10-22 23:10
*/
public class ElasticSearchUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchUtil.class);
private ElasticSearchClient elasticSearchClient;
private BulkProcessor bulkProcessor;
@PostConstruct
public void init() {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {
int numberOfActions = request.numberOfActions();
LOGGER.info("Executing bulk [{}] with {} requests", executionId, numberOfActions);
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
LOGGER.error("Bulk [{}] executed with failures,response = {}", executionId, response.buildFailureMessage());
} else {
LOGGER.info("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis());
}
}
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
LOGGER.error("Failed to execute bulk", failure);
}
};
BulkProcessor bulkProcessor = BulkProcessor.builder(elasticSearchClient.getRestHighLevelClient()::bulkAsync, listener)
// 1000条数据请求执行一次bulk
.setBulkActions(1000)
// 5mb的数据刷新一次bulk
.setBulkSize(new ByteSizeValue(5L, ByteSizeUnit.MB))
// 并发请求数量, 0不并发, 1并发允许执行
.setConcurrentRequests(0)
// 固定1s必须刷新一次
.setFlushInterval(TimeValue.timeValueSeconds(1L))
// 重试5次，间隔1s
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 5))
.build();
this.bulkProcessor = bulkProcessor;
}
@PreDestroy
public void destroy() {
try {
bulkProcessor.awaitClose(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
LOGGER.error("Failed to close bulkProcessor", e);
}
LOGGER.info("bulkProcessor closed!");
}
/**
* 修改
*
* @param request
* @throws IOException
*/
public void update(UpdateRequest request) {
this.bulkProcessor.add(request);
}
/**
* 新增
*
* @param request
*/
public void insert(IndexRequest request) {
this.bulkProcessor.add(request);
}
/**
* Setter method for property elasticSearchClient.
*
* @param elasticSearchClient value to be assigned to property elasticSearchClient
*/
public void setElasticSearchClient(ElasticSearchClient elasticSearchClient) {
this.elasticSearchClient = elasticSearchClient;
}
}
5.junit测试
import xx.Application;
import xx.config.es.ElasticSearchClient;
import xx.config.es.ElasticSearchUtil;
import org.elasticsearch.action.get.GetResponse;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.io.IOException;
/**
* @author laimailai
* @version EsTest.java, v 0.1 2018-10-22 15:36
*/
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest(classes = Application.class)
@ActiveProfiles("dev")
public class EsTest {
String index = "index";
String type = "type";
@Autowired
private ElasticSearchUtil elasticSearchUtil;
@Qualifier("elasticSearchClient")
@Autowired
private ElasticSearchClient elasticSearchClient;
@Qualifier("elasticSearchBusinessClient")
@Autowired
private ElasticSearchClient elasticSearchBusinessClient;
/**
* 主库根据id查询测试
*
* @throws IOException
*/
@Test
public void getByIdTest() throws IOException {
GetResponse response = elasticSearchClient.getDocument(index, type, "1");
System.out.println(response);
}
/**
* 业务库根据id查询测试
*
* @throws IOException
*/
@Test
public void getByIdBusinessTest() throws IOException {
GetResponse response = elasticSearchBusinessClient.getDocument(index, type, "1");
System.out.println(response);
}
}