spring boot集成 elasticsearch集群 之rest high level client
四种方式
操作es的主要有四种方式:
第一种 :spring date es ,这种方式简单,多用于一些简单业务,但因为方法都帮你封装好了,灵活度不高!(复杂业务不推荐)
第二种 :transportClient ,这种方式，官方已经明确表示在ES 7.0版本中将弃用TransportClient客户端，且在8.0版本中完全移除它
第三种 :REST Client 这种方式是基于http 与 es 通信,方便(官网推荐),主要有restHighLevelClient 和restlowLevelClient俩版本,这里也是我是选用的方式.
第四种：Http restful api接口调用，elaticsearch-head常用方式
依赖
4.0.0
org.springframework.boot
spring-boot-starter-parent
2.1.9.RELEASE
com.yidu
yangguangetl
0.0.1-SNAPSHOT
yangguangetl
ETL project to transfer data from mysql to es and mysql
1.8
org.springframework.boot
spring-boot-starter
org.springframework.boot
spring-boot-starter-test
test
org.elasticsearch
elasticsearch
6.8.3
com.alibaba
fastjson
1.2.61
org.elasticsearch.client
elasticsearch-rest-high-level-client
6.8.3
org.elasticsearch.client
elasticsearch-rest-client
6.8.3
org.projectlombok
lombok
1.18.10
provided
commons-lang
commons-lang
2.6
com.alibaba
fastjson
1.2.54
org.springframework.boot
spring-boot-maven-plugin
application.properties 配置
#集群用","连接
elasticSearch.hostlist=127.0.0.1:9200
elasticSearch.client.connectNum=10
elasticSearch.client.connectPerRoute=50配置REST Client
package com.mqm501.es;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
/**
* Project: sunshineetl Package: com.mqm501.es
* Create Time: 2019年10月12日 17:13
* @author: qingmei.meng
* Description:
**/
@Configuration
@ComponentScan(basePackageClasses=ESClientSpringFactory.class)
public class ElasticsearchConfig {
@Value("${elasticSearch.client.connectNum}")
private Integer connectNum;
@Value("${elasticSearch.client.connectPerRoute}")
private Integer connectPerRoute;
@Value("${elasticSearch.hostlist}")
private String hostlist;
@Bean
public HttpHost[] httpHost(){
//解析hostlist配置信息
String[] split = hostlist.split(",");
//创建HttpHost数组，其中存放es主机和端口的配置信息
HttpHost[] httpHostArray = new HttpHost[split.length];
for(int i=0;ipackage com.mqm501.es;
import java.io.IOException;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
/**
* Project: sunshineetl Package: com.mqm501.es
* Create Time: 2019年10月12日 17:15
* @author: qingmei.meng
* Description:
**/
public class ESClientSpringFactory {
public static int CONNECT_TIMEOUT_MILLIS = 1000;
public static int SOCKET_TIMEOUT_MILLIS = 30000;
public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
public static int MAX_CONN_PER_ROUTE = 10;
public static int MAX_CONN_TOTAL = 30;
private static HttpHost[] HTTP_HOST;
private RestClientBuilder builder;
private RestClient restClient;
private RestHighLevelClient restHighLevelClient;
private static ESClientSpringFactory esClientSpringFactory = new ESClientSpringFactory();
private ESClientSpringFactory(){}
public static ESClientSpringFactory build(HttpHost[] httpHostArray,
Integer maxConnectNum, Integer maxConnectPerRoute){
HTTP_HOST = httpHostArray;
MAX_CONN_TOTAL = maxConnectNum;
MAX_CONN_PER_ROUTE = maxConnectPerRoute;
return esClientSpringFactory;
}
public static ESClientSpringFactory build(HttpHost[] httpHostArray,Integer connectTimeOut, Integer socketTimeOut,
Integer connectionRequestTime,Integer maxConnectNum, Integer maxConnectPerRoute){
HTTP_HOST = httpHostArray;
CONNECT_TIMEOUT_MILLIS = connectTimeOut;
SOCKET_TIMEOUT_MILLIS = socketTimeOut;
CONNECTION_REQUEST_TIMEOUT_MILLIS = connectionRequestTime;
MAX_CONN_TOTAL = maxConnectNum;
MAX_CONN_PER_ROUTE = maxConnectPerRoute;
return esClientSpringFactory;
}
public void init(){
builder = RestClient.builder(HTTP_HOST);
setConnectTimeOutConfig();
setMutiConnectConfig();
restClient = builder.build();
restHighLevelClient = new RestHighLevelClient(builder);
System.out.println("init factory");
}
// 配置连接时间延时
public void setConnectTimeOutConfig(){
builder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS);
requestConfigBuilder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS);
requestConfigBuilder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS);
return requestConfigBuilder;
});
}
// 使用异步httpclient时设置并发连接数
public void setMutiConnectConfig(){
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnTotal(MAX_CONN_TOTAL);
httpClientBuilder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE);
return httpClientBuilder;
});
}
public RestClient getClient(){
return restClient;
}
public RestHighLevelClient getRhlClient(){
return restHighLevelClient;
}
public void close() {
if (restClient != null) {
try {
restClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
System.out.println("close client");
}
}
配置esUtils
package com.mqm501.es;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mqm501.es.SpringBeanUtil;
import com.mqm501.es.entity.es.ESCustomer;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.IndicesClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
/**
* Project: sunshineetl Package: mqm501.es
* Create Time: 2019年10月12日 17:19
* @author: qingmei.meng
* Description:
**/
public class EsUtils {
private static RestHighLevelClient client;
public static RestHighLevelClient getInstance() {
if (client == null) {
synchronized (RestHighLevelClient.class) {
if (client == null) {
client = (RestHighLevelClient) SpringBeanUtil.getBean("restHighLevelClient");
System.out.println(client);
}
}
}
return client;
}
public static void main(String[] args) {
getInstance();
}
/**
* 创建索引
*
* @param index 索引名称
* @param type 类型名称,可传入多个type逗号分隔
* @param arg1 分片数
* @param arg2 副本数
* @param source 需要设置的映射(配置是否分词,是否索引等)
* @return boolean
*/
public static boolean addIndex(String index, String type, int arg1, int arg2, String source) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
//设置索引参数
createIndexRequest.settings(
Settings.builder().put("number_of_shards", arg1).put("number_of_replicas", arg2));
//操作索引的客户端
createIndexRequest.mapping(type, source, XContentType.JSON);
IndicesClient indices = getInstance().indices();
//执行创建
CreateIndexResponse createIndexResponse = null;
try {
indices.create(createIndexRequest);
} catch (IOException e) {
e.printStackTrace();
}
//得到响应
return true;
}
/**
* 判断索引是否存在
* @param index
* @return
* @throws IOException
*/
public static boolean existsIndex(String index) throws IOException {
GetIndexRequest request = new GetIndexRequest(index);
//request.indices(index);
//操作索引的客户端
IndicesClient indices = getInstance().indices();
boolean exists = indices.exists(request, RequestOptions.DEFAULT);
System.out.println("existsIndex: " + exists);
return exists;
}
/**
* 删除索引
*
* @param index 索引名称
* @return boolean
*/
public static boolean delIndex(String index) throws IOException {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index);
//操作索引的客户端
IndicesClient indices = getInstance().indices();
//执行删除
AcknowledgedResponse deleteIndexResponse = indices.delete(deleteIndexRequest,RequestOptions.DEFAULT);
//得到响应
return deleteIndexResponse.isAcknowledged();
}
/**
* 添加数据
*
* @param index 索引名称
* @param type 类型
* @param jsonObject json对象(要添加的数据)
* @return boolean
*/
public static boolean addDoc(String index, String type, String id, JSONObject jsonObject) {
//索引请求对象
IndexRequest indexRequest = new IndexRequest(index, type, id);
indexRequest.source(jsonObject);
//索引响应对象
try {
getInstance().index(indexRequest);
} catch (IOException e) {
e.printStackTrace();
}
//获取响应结果
return true;
}
/**
* 修改数据
*
* @param index 索引名称
* @param type 类型
* @param jsonObject json对象(要添加的数据)
* @param id
* @return RestStatus ok
*/
public static boolean updateDoc(String index, String type, JSONObject jsonObject, String id) {
UpdateRequest updateRequest = new UpdateRequest(index, type, id);
updateRequest.doc(jsonObject);
//索引响应对象
UpdateResponse update = null;
try {
getInstance().update(updateRequest);
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
/**
* 获取记录信息
* @param index
* @param type
* @param id
* @throws IOException
*/
public static void getDoc(String index, String type, String id) {
GetIndexRequest getRequest = new GetIndexRequest(index, type, id.toString());
//操作索引的客户端
IndicesClient indices = getInstance().indices();
GetIndexResponse getResponse = null;
try {
getResponse = indices.get(getRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("get: " + JSON.toJSONString(getResponse));
}
/**
* 删除数据
*
* @param index 索引名称
* @param type 类型
* @param id 文档id(即某条数据的id)
* @return RestStatus ok
*/
public static boolean deleteDoc(String index, String type, String id) {
DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
//索引响应对象
try {
getInstance().delete(deleteRequest);
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
/**
* 使用分词查询
*
* @param index 索引名称
* @param type 类型
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @param query 查询条件
* @param sortField 排序字段(倒序)
* @return List