Java连接ElasticSearch（springboot2.X配置ES连接的方式）
springboot2.X配置ES连接的3种方式
Java连接ElasticSearch
Java连接ElasticSearch的2种方式
有两种方式：
一种 是9200端口（RestClient）rest 接口，基于http协议；
另一种是用 节点的9300端口（TransportClient），基于Tcp协议；（不推荐使用，理由如下）
es7.X 版本的改变
简单点说就是 TransportClient 在es7后不支持了,所以你要用 java high level rest client，这个是用http的请求)
所以 ，后面都 是采用9200端口的方式！！！
es6.X 以下的版本
elasticsearch 官网推荐是使用基于http协议的restClient去充当客户端连接ES，
如果想基于TCP协议，9300端口从传输层获取es文档数据也是可以的；但es7以上版本不支持
SpringBoot整合ES的三种方式（API、REST Client、Data-ES）
这种方式，官方已经明确表示在ES 7.0版本中将弃用TransportClient客户端，且在8.0版本中完全移除它，案例ES版本已经是6.5.0了。
版本搭配：SpringBoot 2.1.3，ES 6.5.4
1.整合案例一（使用Java API，基于TCP传输层协议，端口9300）
1.1 pom文件（gradle文件）依赖如下：
（2017年9月，Spring5.0版本发布以后，自2018年Pivotal公司放弃maven，转而用gradle托管spring源码；）
plugins {
id 'org.springframework.boot' version '2.1.3.RELEASE'
id 'java'
}
apply plugin: 'io.spring.dependency-management'
group = 'com.yiche'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = '1.8'
repositories {
mavenCentral()
mavenLocal()
maven{ url 'http://maven.aliyun.com/nexus/content/groups/public/'}
}
configurations {
// providedRuntime
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter-web'
// providedRuntime('org.springframework.boot:spring-boot-starter-tomcat')
testImplementation 'org.springframework.boot:spring-boot-starter-test'
compile 'javax.servlet:javax.servlet-api:4.0.1'
compile 'org.elasticsearch:elasticsearch:6.5.4'
compile('org.elasticsearch.client:transport:6.5.4') {
exclude(module: 'org.elasticsearch:elasticsearch')
}
compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:6.5.4'
compile 'org.elasticsearch.plugin:transport-netty4-client:6.5.4'
compile 'org.apache.commons:commons-io:1.3.2'
}
1.2配置文件
@Configuration
public class ESConfig {
@Bean
public TransportClient client() throws UnknownHostException {
/**
* 指定集群名称,如果改了集群名,这里一定要加
* 6.0以下版本默认为elasticsearch
* 6.0以上版本，es6.5.4
* /
Settings settings = Settings.builder()
.put("cluster.name", "alibabaProductCluster")
.build();
TransportClient client = new PreBuiltTransportClient(settings);
/**
*ES的TCP端口为9300,而不是之前练习的HTTP端口9200
* 如果你的es是6.0以上版本，你是找不到InetSocketTransportAddress这个类的；
* 用TransportAddress ，如下行代码；
*TransportAddress node = new TransportAddress(newInetSocketAddress(host, port)；
*去替换变量node的定义
*/
InetSocketTransportAddress node = new InetSocketTransportAddress(
InetAddress.getByName("localhost"),
9300
);
client.addTransportAddress(node);
return client;
}
}
1.3 利用TransportClient 实现业务查询
/**
* 根据id查询
*/
@RequestMapping(value = "/book", method = RequestMethod.GET)
public ResponseEntity> get(@RequestParam("id") String id) {
GetResponse result = client.prepareGet("book", "novel", id).get();
return new ResponseEntity<>(result.getSource(), HttpStatus.OK);
}
/**
* 添加文档
*
* @param id book id
* @param name book name
*/
@RequestMapping(value = "/book", method = RequestMethod.POST)
public ResponseEntity add(@RequestParam("id") String id, @RequestParam("name") String name) {
try {
// 构造ES的文档，这里注意startObject()开始构造，结束构造一定要加上endObject()
XContentBuilder content = XContentFactory.jsonBuilder().startObject().
field("id", id)
.field("name", name)
.endObject();
IndexResponse result = client.prepareIndex("book", "novel")
.setSource(content).get();
return new ResponseEntity<>(result.getId(), HttpStatus.OK);
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
/**
* 根据id删除book
*
* @param id book id
*/
@RequestMapping(value = "/book/{id}", method = RequestMethod.DELETE)
public ResponseEntity delete(@PathVariable(value = "id") String id) {
DeleteResponse result = client.prepareDelete("book", "novel", id).get();
return new ResponseEntity<>(result.getResult().toString(), HttpStatus.OK);
}
/**
* 更新文档，这里的Book可以不管他，这样做是为了解决PUT请求的问题，随便搞
*/
@RequestMapping(value = "/book", method = RequestMethod.PUT)
public ResponseEntity update(@RequestBody Book book) {
System.out.println(book);
// 根据id查询
UpdateRequest updateRequest = new UpdateRequest("book", "novel", book.getId().toString());
try {
XContentBuilder contentBuilder = XContentFactory.jsonBuilder().startObject();
if (StringUtils.isNotBlank(book.getName())) {
contentBuilder.field("name", book.getName());
}
contentBuilder.endObject();
updateRequest.doc(contentBuilder);
} catch (IOException e) {
e.printStackTrace();
}
// 进行更新
UpdateResponse updateResponse = new UpdateResponse();
try {
updateResponse = client.update(updateRequest).get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
return new ResponseEntity<>(updateResponse.getResult().toString(), HttpStatus.OK);
}
}
整合案例二(基于http协议，使用RestClinet)—>ES官方推荐使用这种方式
2.1 引入依赖
elasticsearch_data
com.glodon
1.0-SNAPSHOT
4.0.0
es_java_rest
org.elasticsearch.client
elasticsearch-rest-client
6.5.4
org.json
json
20160810
org.apache.maven.plugins
maven-shade-plugin
3.1.0
package
shade
org.apache.http
hidden.org.apache.http
org.apache.logging
hidden.org.apache.logging
org.apache.commons.codec
hidden.org.apache.commons.codec
org.apache.commons.logging
hidden.org.apache.commons.logging
2.2 配置文件
config基本配置如下
@Configuration
public class RestConfig {
@Bean
public RestClient getClient() throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException {
// 如果有多个从节点可以持续在内部new多个HttpHost，参数1是ip,参数2是HTTP端口，参数3是通信协议
RestClientBuilder clientBuilder = RestClient.builder(new HttpHost("localhost", 9200, "http"));
// 1.设置请求头
Header[] defaultHeaders = {new BasicHeader("header", "value")};
clientBuilder.setDefaultHeaders(defaultHeaders);
//
//2. 设置超时时间，多次尝试同一请求时应该遵守的超时。默认值为30秒，与默认套接字超时相同。若自定义套接字超时，则应相应地调整最大重试超时
clientBuilder.setMaxRetryTimeoutMillis(60000);
/**
*3.设置失败监听器，
*每次节点失败都可以监听到，可以作额外处理
*/
clientBuilder.setFailureListener(new RestClient.FailureListener() {
@Override
public void onFailure(Node node) {
super.onFailure(node);
System.out.println(node.getName() + "==节点失败了");
}
});
/** 4.配置节点选择器，客户端以循环方式将每个请求发送到每一个配置的节点上，
*发送请求的节点，用于过滤客户端，将请求发送到这些客户端节点，默认向每个配置节点发送，
*这个配置通常是用户在启用嗅探时向专用主节点发送请求（即只有专用的主节点应该被HTTP请求命中）
*/
clientBuilder.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
/*
*5. 配置异步请求的线程数量，Apache Http Async Client默认启动一个调度程序线程，以及由连接管理器使用的许多工作线程
*（与本地检测到的处理器数量一样多，取决于Runtime.getRuntime().availableProcessors()返回的数量）。线程数可以修改如下,
*这里是修改为1个线程，即默认情况
*/
clientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
return httpAsyncClientBuilder.setDefaultIOReactorConfig(
IOReactorConfig.custom().setIoThreadCount(1).build()
);
}
});
/**
*6. 配置连接超时和套接字超时
*配置请求超时，将连接超时（默认为1秒）和套接字超时（默认为30秒）增加，
*这里配置完应该相应地调整最大重试超时（默认为30秒），即上面的setMaxRetryTimeoutMillis，一般于最大的那个值一致即60000
*/
clientBuilder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
// 连接5秒超时，套接字连接60s超时
return requestConfigBuilder.setConnectTimeout(5000).setSocketTimeout(60000);
}
});
// 最后配置好的clientBuilder再build一下即可得到真正的Client
return clientBuilder.build();
}
}
2.2.0 ES配置通信加密（2018年ElasticSearch全球遭黑客攻击，ES官方解说重要数据需要配置加密）
/*
配置通信加密，有多种方式：setSSLContext、setSSLSessionStrategy和setConnectionManager(它们的重要性逐渐递增)
*/
KeyStore truststore = KeyStore.getInstance("jks");
try (InputStream is = Files.newInputStream(keyStorePath)) {
truststore.load(is, keyStorePass.toCharArray());
}
SSLContextBuilder sslBuilder = SSLContexts.custom().loadTrustMaterial(truststore, null);
final SSLContext sslContext = sslBuilder.build();
clientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
return httpClientBuilder.setSSLContext(sslContext);
}
});
2.2.1 RestClientBuilder中的自定义节点选择器的需求配置（–>大数据中是要求配置机架感知的）
// 进行详细的配置
clientBuilder.setNodeSelector(new NodeSelector() {
// 设置分配感知节点选择器，允许选择本地机架中的节点（如果有），否则转到任何机架中的任何其他节点。
@Override
public void select(Iterable nodes) {
boolean foundOne = false;
for (Node node: nodes) {
String rackId = node.getAttributes().get("rack_id").get(0);
if ("rack_one".equals(rackId)) {
foundOne = true;
break;
}
}
if (foundOne) {
Iterator nodesIt = nodes.iterator();
while (nodesIt.hasNext()) {
Node node = nodesIt.next();
String rackId = node.getAttributes().get("rack_id").get(0);
if ("rack_one".equals(rackId) == false) {
nodesIt.remove();
}
}
}
}
});
2.2.2 配置ES安全认证（如果你的ES集群设置了密码的话，或者密钥的话）
/*
如果ES设置了密码，那这里也提供了一个基本的认证机制，下面设置了ES需要基本身份验证的默认凭据提供程序
*/
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials("user", "password"));
clientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
});
/*
上面采用异步机制实现抢先认证，这个功能也可以禁用，这意味着每个请求都将在没有授权标头的情况下发送，然后查看它是否被接受，
并且在收到HTTP 401响应后，它再使用基本认证头重新发送完全相同的请求，这个可能是基于安全、性能的考虑
*/
clientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
// 禁用抢先认证的方式
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
});
2.3 业务controller
@RestController
@RequestMapping("/rest/book")
public class BookController {
@Autowired
private RestClient client;
// // RequestOptions类保存应在同一应用程序中的多个请求之间共享的部分请求
// private static final RequestOptions COMMON_OPTIONS;
//
// static {
// RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
// // 添加所有请求所需的任何标头。
// builder.addHeader("Authorization", "Bearer " + TOKEN);
// // 自定义响应使用者
// builder.setHttpAsyncResponseConsumerFactory(
// new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
// COMMON_OPTIONS = builder.build();
// }
@RequestMapping(value = "/go", method = RequestMethod.GET)
public ResponseEntity go() {
return new ResponseEntity<>("go", HttpStatus.OK);
}
/**
* 同步执行HTTP请求
* @return
* @throws IOException
*/
@RequestMapping(value = "/es", method = RequestMethod.GET)
public ResponseEntity getEsInfo() throws IOException {
// 构造HTTP请求，第一个参数是请求方法，第二个参数是服务器的端点，host默认是http://localhost:9200
Request request = new Request("GET", "/");
// // 设置其他一些参数比如美化json
// request.addParameter("pretty", "true");
// // 设置请求体
// request.setEntity(new NStringEntity("{\"json\":\"text\"}", ContentType.APPLICATION_JSON));
// // 还可以将其设置为String，默认为ContentType为application/json
// request.setJsonEntity("{\"json\":\"text\"}");
/*
performRequest是同步的，将阻塞调用线程并在请求成功时返回Response，如果失败则抛出异常
内部属性可以取出来通过下面的方法
*/
Response response = client.performRequest(request);
// // 获取请求行
// RequestLine requestLine = response.getRequestLine();
// // 获取host
// HttpHost host = response.getHost();
// // 获取状态码
// int statusCode = response.getStatusLine().getStatusCode();
// // 获取响应头
// Header[] headers = response.getHeaders();
// 获取响应体
String responseBody = EntityUtils.toString(response.getEntity());
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
/**
* 异步执行HTTP请求
* @return
*/
@RequestMapping(value = "/es/asyn", method = RequestMethod.GET)
public ResponseEntity asynchronous() {
Request request = new Request(
"GET",
"/");
client.performRequestAsync(request, new ResponseListener() {
@Override
public void onSuccess(Response response) {
System.out.println("异步执行HTTP请求并成功");
}
@Override
public void onFailure(Exception exception) {
System.out.println("异步执行HTTP请求并失败");
}
});
return null;
}
/**
* 并行异步执行HTTP请求
*/
@RequestMapping(value = "/ps", method = RequestMethod.POST)
public void parallAsyn(@RequestBody Book[] documents) {
// final CountDownLatch latch = new CountDownLatch(documents.length);
// for (int i = 0; i < documents.length; i++) {
// Request request = new Request("PUT", "/posts/doc/" + i);
// //let's assume that the documents are stored in an HttpEntity array
// request.setEntity(documents[i]);
// client.performRequestAsync(
// request,
// new ResponseListener() {
// @Override
// public void onSuccess(Response response) {
//
// latch.countDown();
// }
//
// @Override
// public void onFailure(Exception exception) {
//
// latch.countDown();
// }
// }
// );
// }
// latch.await();
}
/**
* 添加ES对象, Book的ID就是ES中存储的document的ID，所以最好不要为空，自定义生成的ID太浮夸
*
* @return ResponseEntity
* @throws IOException
*/
@RequestMapping(value = "/", method = RequestMethod.POST)
public ResponseEntity add(@RequestBody Book book) throws IOException {
// 构造HTTP请求，第一个参数是请求方法，第二个参数是服务器的端点，host默认是http://localhost:9200，
// endpoint直接指定为index/type的形式
Request request = new Request("POST", new StringBuilder("/book/book/").
append(book.getId()).toString());
// 设置其他一些参数比如美化json
request.addParameter("pretty", "true");
JSONObject jsonObject = new JSONObject(book);
System.out.println(jsonObject.toString());
// 设置请求体并指定ContentType，如果不指定默认为APPLICATION_JSON
request.setEntity(new NStringEntity(jsonObject.toString()));
// 发送HTTP请求
Response response = client.performRequest(request);
// 获取响应体, id: AWXvzZYWXWr3RnGSLyhH
String responseBody = EntityUtils.toString(response.getEntity());
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
/**
* 根据id获取ES对象
*
* @param id
* @return
* @throws IOException
*/
@RequestMapping(value = "/{id}", method = RequestMethod.GET)
public ResponseEntity getBookById(@PathVariable("id") String id) {
Request request = new Request("GET", new StringBuilder("/book/book/").
append(id).toString());
// 添加json返回优化
request.addParameter("pretty", "true");
Response response = null;
String responseBody = null;
try {
// 执行HHTP请求
response = client.performRequest(request);
responseBody = EntityUtils.toString(response.getEntity());
} catch (IOException e) {
return new ResponseEntity<>("can not found the book by your id", HttpStatus.NOT_FOUND);
}
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
/**
* 根据id更新Book
*
* @param id
* @param book
* @return
*/
@RequestMapping(value = "/{id}", method = RequestMethod.PUT)
public ResponseEntity updateBook(@PathVariable("id") String id, @RequestBody Book book) throws IOException {
// 构造HTTP请求
Request request = new Request("POST", new StringBuilder("/book/book/").
append(id).append("/_update").toString());
request.addParameter("pretty", "true");
// 将数据丢进去，这里一定要外包一层“doc”，否则内部不能识别
JSONObject jsonObject = new JSONObject();
jsonObject.put("doc", new JSONObject(book));
request.setEntity(new NStringEntity(jsonObject.toString()));
// 执行HTTP请求
Response response = client.performRequest(request);
// 获取返回的内容
String responseBody = EntityUtils.toString(response.getEntity());
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
/**
* 使用脚本更新Book
* @param id
* @param
* @return
* @throws IOException
*/
@RequestMapping(value = "/update2/{id}", method = RequestMethod.PUT)
public ResponseEntity updateBook2(@PathVariable("id") String id, @RequestParam("name") String name) throws IOException {
// 构造HTTP请求
Request request = new Request("POST", new StringBuilder("/book/book/").
append(id).append("/_update").toString());
request.addParameter("pretty", "true");
JSONObject jsonObject = new JSONObject();
// 创建脚本语言，如果是字符变量，必须加单引号
StringBuilder op1 = new StringBuilder("ctx._source.name=").append("'" + name + "'");
jsonObject.put("script", op1);
request.setEntity(new NStringEntity(jsonObject.toString(), ContentType.APPLICATION_JSON));
// 执行HTTP请求
Response response = client.performRequest(request);
// 获取返回的内容
String responseBody = EntityUtils.toString(response.getEntity());
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
@RequestMapping(value = "/{id}", method = RequestMethod.DELETE)
public ResponseEntity deleteById(@PathVariable("id") String id) throws IOException {
Request request = new Request("DELETE", new StringBuilder("/book/book/").append(id).toString());
request.addParameter("pretty", "true");
// 执行HTTP请求
Response response = client.performRequest(request);
// 获取结果
String responseBody = EntityUtils.toString(response.getEntity());
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
}
3. 整合案例三（使用Spring-data-es）
除了上述方式，Spring也提供了本身基于SpringData实现的一套方案spring-data-elasticsearch，
spring的这套操作ES的工具包，也是基于TCP的TransportClient来封装的，配置端口9300（未来ES7.X不支持，不推荐使用）
版本之间的搭配建议为：
| spring data elasticsearch | elasticsearch |
|---|---|
| 3.1.x | 6.2.2 |
| 3.0.x | 5.5.0 |
3.1 引入依赖
elasticsearch_data
com.glodon
1.0-SNAPSHOT
4.0.0
es_java_data
org.springframework.data
spring-data-elasticsearch
3.0.10.RELEASE
spring-libs-release
Spring Releases
https://repo.spring.io/libs-release
false
3.2 配置Data ES
spring:
data:
elasticsearch:
cluster-nodes: localhost:9300 # 配置IP及端口号
cluster-name: elasticsearch
# cluster-name: elasticsearch622 # 配置集群名，默认为elasticsearch，如果手动更改过，这里一定要指定
# repositories:
# enabled: true
3.3 创建ElasticsearchRepository实例对象
public interface BookRepository extends ElasticsearchRepository {
Book findByName(String name);
List findByAuthor(String author);
Book findBookById(String id);
}
3.4 配置Repositories扫描
/**
* @description 基础包的注释驱动配置，配置自动扫描的repositories根目录
*/
@Configuration
@EnableElasticsearchRepositories(basePackages = "com.glodon.repositories")
public interface ESConfig {
}
3.5 增、删、改、查业务操作
import com.glodon.models.Book;
import com.glodon.repositories.BookRepository;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
/**
* @author liuwg-a
* @date 2018/9/18 10:43
* @description
*/
@RequestMapping("/book")
@RestController
public class BookController {
@Autowired
BookRepository bookRepository;
@RequestMapping(value = "/add_index", method = RequestMethod.POST)
public ResponseEntity indexDoc(@RequestBody Book book) {
System.out.println("book===" + book);
bookRepository.save(book);
return new ResponseEntity<>("save executed!", HttpStatus.OK);
}
@RequestMapping(value = "/", method = RequestMethod.GET)
public ResponseEntity getAll() {
Iterable all = bookRepository.findAll();
return new ResponseEntity<>(all, HttpStatus.OK);
}
@RequestMapping(value = "/{name}", method = RequestMethod.GET)
public ResponseEntity getByName(@PathVariable("name") String name) {
Book book = bookRepository.findByName(name);
return new ResponseEntity<>(book, HttpStatus.OK);
}
@RequestMapping(value = "/{id}", method = RequestMethod.PUT)
public ResponseEntity updateBook(@PathVariable("id") String id,
@RequestBody Book updateBook) {
Book book = bookRepository.findBookById(id);
if (StringUtils.isNotBlank(updateBook.getId())) {
book.setId(updateBook.getId());
}
if (StringUtils.isNotBlank(updateBook.getName())) {
book.setName(updateBook.getName());
}
if (StringUtils.isNotBlank(updateBook.getAuthor())) {
book.setAuthor(updateBook.getAuthor());
}
bookRepository.save(book);
return new ResponseEntity<>(book, HttpStatus.OK);
}
@RequestMapping(value = "/{id}", method = RequestMethod.DELETE)
public ResponseEntity deleteBook(@PathVariable("id") String id) {
bookRepository.deleteById(id);
return new ResponseEntity<>("delete execute!", HttpStatus.OK);
}
}
ES官方对于不同语言操作提供了不同客户端，上述只是入门的简单操作，具体参看Elasticsearch Clients。