【工作记录】springboot集成spring-data-elasticsearch访问es及问题解决
背景
前文我们介绍了通过可视化爬虫爬取新闻到Mysql库、基于docker-compose的elk集群部署及抽取mysql数据到es的过程，本文我们介绍通过集成springboot和spring-data-elasticsearch客户端完成连接es并查询数据的开发过程以及遇到的问题和解决方案，希望对大家能有所帮助，对文中内容有任何疑问或者建议欢迎留言或者私信~
官网文档地址: 官方文档直达
官方提供的版本对应关系:
| Spring Data Release Train | Spring Data Elasticsearch | Elasticsearch | Spring Framework | Spring Boot |
|---|---|---|---|---|
| 2022.0 (Turing) | 5.0.x | 8.5.3 | 6.0.x | 3.0.x |
| 2021.2 (Raj) | 4.4.x | 7.17.3 | 5.3.x | 2.7.x |
| 2021.1 (Q) | 4.3.x | 7.15.2 | 5.3.x | 2.6.x |
| 2021.0 (Pascal) | 4.2.x[1] | 7.12.0 | 5.3.x | 2.5.x |
| 2020.0 (Ockham)[1] | 4.1.x[1] | 7.9.3 | 5.3.2 | 2.4.x |
| Neumann[1] | 4.0.x[1] | 7.6.2 | 5.2.12 | 2.3.x |
| Moore[1] | 3.2.x[1] | 6.8.12 | 5.2.12 | 2.2.x |
| Lovelace[1] | 3.1.x[1] | 6.2.2 | 5.1.19 | 2.1.x |
| Kay[1] | 3.0.x[1] | 5.5.0 | 5.0.13 | 2.0.x |
| Ingalls[1] | 2.1.x[1] | 2.4.0 | 4.3.25 | 1.5.x |
文中使用到的各个版本
| 名称 | 版本号 |
|---|---|
| springboot | 2.7.8 |
| spring-data-elasticsearch | 4.4.7 |
| spring | 5.3.25 |
| elasticsearch-java | 7.17.8 |
| elasticsearch-rest-client | 7.17.8 |
| 部署的elasticsearch版本 | 8.6.2 |
开发过程
1. 新建springboot + maven项目，引入相关依赖
pom文件如下:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0modelVersion>
<packaging>pompackaging>
<parent>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-parentartifactId>
<version>2.7.8version>
<relativePath/>
parent>
<groupId>com.zjtx.techgroupId>
<artifactId>es-demoartifactId>
<version>0.0.1version>
<dependencies>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-webartifactId>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-configuration-processorartifactId>
<optional>trueoptional>
dependency>
<dependency>
<groupId>org.projectlombokgroupId>
<artifactId>lombokartifactId>
<optional>trueoptional>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-autoconfigureartifactId>
<version>2.7.8version>
dependency>
<dependency>
<groupId>org.springframework.datagroupId>
<artifactId>spring-data-elasticsearchartifactId>
<exclusions>
<exclusion>
<artifactId>jakarta.json-apiartifactId>
<groupId>jakarta.jsongroupId>
exclusion>
exclusions>
dependency>
<dependency>
<groupId>jakarta.jsongroupId>
<artifactId>jakarta.json-apiartifactId>
<version>2.0.1version>
dependency>
<dependency>
<groupId>org.elasticsearchgroupId>
<artifactId>elasticsearch-x-contentartifactId>
<version>7.17.8version>
dependency>
<dependency>
<groupId>io.github.hakky54groupId>
<artifactId>sslcontext-kickstartartifactId>
<version>7.1.0version>
dependency>
dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-compiler-pluginartifactId>
<version>3.1version>
<configuration>
<source>1.8source>
<target>1.8target>
<encoding>UTF-8encoding>
configuration>
plugin>
plugins>
build>
project>
2. 添加配置类及配置文件
新建config包，创建EsClientProperties类，此文件与application.yml文件中的es配置一一对应
package com.zjtx.tech.maxkey.demo.es8.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* @author 泽济天下
* @date 2023年03月13日15:32
* @description:
*/
@Data
@Configuration
@ConfigurationProperties(prefix = "es.client")
public class EsClientProperties {
private boolean enabled;
private String host;
private String port;
private String username;
private String password;
}
修改application.yml文件
server:
port: 8888
es:
client:
enabled: true
host: 172.16.10.221
port: 9200
username: codevisitor
password: cnhqd@1234
3. 添加ES客户端配置Bean
package com.zjtx.tech.maxkey.demo.es8.config;
import nl.altindag.ssl.SSLFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchConfiguration;
@Configuration
@ConditionalOnProperty(prefix = "es.client", name = "enabled", havingValue = "true")
public class MyClientConfig extends ElasticsearchConfiguration {
@Autowired
private EsClientProperties esClientProperties;
@Override
public ClientConfiguration clientConfiguration() {
//FIX 修改SSL证书校验失败的问题
SSLFactory sslFactory = SSLFactory.builder()
.withUnsafeTrustMaterial()
.withUnsafeHostnameVerifier()
.build();
return ClientConfiguration.builder()
.connectedTo(esClientProperties.getHost(), esClientProperties.getPort())
.usingSsl(sslFactory.getSslContext(), sslFactory.getHostnameVerifier())
.withBasicAuth(esClientProperties.getUsername(), esClientProperties.getPassword())
.build();
}
}
4. 添加controller进行测试
package com.zjtx.tech.maxkey.demo.es8.controller;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import com.zjtx.tech.maxkey.demo.es8.model.YdstuNews;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @author 泽济天下
* @date 2023年03月13日15:39
* @description:
*/
@RestController
@RequestMapping("esClient")
public class EsClientController {
@Autowired
private ElasticsearchClient esClient;
@Autowired
private ElasticsearchOperations operations;
@GetMapping("getById")
public String getById(String id){
YdstuNews news = operations.get(id, YdstuNews.class);
if(news != null) {
System.out.println("news = " + news);
} else {
System.out.println("没有找到对应数据");
}
return "success";
}
}
运行项目并访问http://localhost:8888/esClient/getById?id=1和http://localhost:8888/esClient/getById?id=c1281a144535
可以看到控制台打印如下内容:
2023-03-14 10:07:38.685 INFO 10680 --- [ restartedMain] o.s.b.w.embedded.tomcat.TomcatWebServer : Tomcat started on port(s): 8888 (http) with context path ''
2023-03-14 10:07:38.693 INFO 10680 --- [ restartedMain] c.zjtx.tech.maxkey.demo.DemoApplication : Started DemoApplication in 2.214 seconds (JVM running for 2.953)
2023-03-14 10:07:44.905 INFO 10680 --- [nio-8888-exec-1] o.apache.tomcat.util.http.parser.Cookie : A cookie header was received [Hm_lvt_ae02bfc0d49b4dfa890f81d96472fe99=1677222282,1677224194,1677467229,1677550739;] that contained an invalid cookie. That cookie will be ignored.
Note: further occurrences of this error will be logged at DEBUG level.
2023-03-14 10:07:44.910 INFO 10680 --- [nio-8888-exec-1] o.a.c.c.C.[Tomcat].[localhost].[/] : Initializing Spring DispatcherServlet 'dispatcherServlet'
2023-03-14 10:07:44.910 INFO 10680 --- [nio-8888-exec-1] o.s.web.servlet.DispatcherServlet : Initializing Servlet 'dispatcherServlet'
2023-03-14 10:07:44.911 INFO 10680 --- [nio-8888-exec-1] o.s.web.servlet.DispatcherServlet : Completed initialization in 1 ms
没有找到对应数据
news = YdstuNews(id=c1281a144535, title=我校学报刊发论文入选中国百篇最具影响国内学术论文, content=<p style="font-family:宋体, simsun;font-size:18px;"> 近日，中国科学技术信息研究所发布2022中国卓越科技论文报告，在引人注目的“中国百篇最具影响国内学术论文”评选中，我校于林平、薛博茹、任效忠、刘鹰等撰写的《单进水管结构对单通道矩形圆弧角养殖池水动力特性的影响研究》（《大连海洋大学学报》2020年1期）成功入选，这也是国内水产类期刊唯一一篇入选论文。</p>
<p style="font-family:宋体, simsun;font-size:18px;"> 据悉，中国科学技术信息研究所从2016年开始发布中国卓越科技论文报告，本年度入选论文的选取范围是2017—2021年中国科技论文与引文数据库(CSTPCD) 所收录的科技论文。评选中取近5年CSTPCD中发表在中国科技核心期刊，且累计被引用次数进入相应发表年度和所属学科领域的前千分之一的论文，作为本年度的候选论文。评选者根据各个学科领域的论文数量规模以及候选论文数量，结合我国科技发展的重点领域和优先主题，参考候选论文的文献类型、基金项目资助情况、被引用分布等方面的情况，从中择优选取“中国百篇最具影响国内学术论文”。本年度共选出100篇论文作为“中国百篇最具影响国内学术论文”，分属于87个机构，其中高等院校61篇，研究机构18篇，100篇论文分布于地学、化学、生物学、水产学等35个学科。</p>
<p style="text-align:center;font-family:宋体, simsun;font-size:18px;">科技处 供稿</p>
<p><br /></p>)
到这里，我们就完成了spring-data-elasticsearch和springboot项目的集成，完成了对es的链接和简单查询。
报错及解决方案记录
1. 找不到JsonProvider类
报错信息:
Caused by: java.lang.NoClassDefFoundError: jakarta/json/spi/JsonProvider
at java.lang.ClassLoader.defineClass1(Native Method) ~[na:1.8.0_181]
at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[na:1.8.0_181]
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[na:1.8.0_181]
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[na:1.8.0_181]
at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[na:1.8.0_181]
at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[na:1.8.0_181]
at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[na:1.8.0_181]
at java.security.AccessController.doPrivileged(Native Method) ~[na:1.8.0_181]
at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[na:1.8.0_181]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_181]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) ~[na:1.8.0_181]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_181]
at co.elastic.clients.json.jackson.JacksonJsonpMapper.<init>(JacksonJsonpMapper.java:49) ~[elasticsearch-java-7.17.8.jar:na]
at co.elastic.clients.json.jackson.JacksonJsonpMapper.<init>(JacksonJsonpMapper.java:56) ~[elasticsearch-java-7.17.8.jar:na]
at org.springframework.data.elasticsearch.client.elc.ElasticsearchClients.getElasticsearchTransport(ElasticsearchClients.java:248) ~[spring-data-elasticsearch-4.4.7.jar:4.4.7]
at org.springframework.data.elasticsearch.client.elc.ElasticsearchClients.createImperative(ElasticsearchClients.java:167) ~[spring-data-elasticsearch-4.4.7.jar:4.4.7]
at org.springframework.data.elasticsearch.client.elc.ElasticsearchConfiguration.elasticsearchClient(ElasticsearchConfiguration.java:73) ~[spring-data-elasticsearch-4.4.7.jar:4.4.7]
at com.zjtx.tech.maxkey.demo.es8.config.MyClientConfig$$EnhancerBySpringCGLIB$$92dcd9a4.CGLIB$elasticsearchClient$3(<generated>) ~[classes/:na]
at com.zjtx.tech.maxkey.demo.es8.config.MyClientConfig$$EnhancerBySpringCGLIB$$92dcd9a4$$FastClassBySpringCGLIB$$e566fcbb.invoke(<generated>) ~[classes/:na]
at org.springframework.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:244) ~[spring-core-5.3.25.jar:5.3.25]
at org.springframework.context.annotation.ConfigurationClassEnhancer$BeanMethodInterceptor.intercept(ConfigurationClassEnhancer.java:331) ~[spring-context-5.3.25.jar:5.3.25]
at com.zjtx.tech.maxkey.demo.es8.config.MyClientConfig$$EnhancerBySpringCGLIB$$92dcd9a4.elasticsearchClient(<generated>) ~[classes/:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_181]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_181]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_181]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_181]
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:154) ~[spring-beans-5.3.25.jar:5.3.25]
... 39 common frames omitted
Caused by: java.lang.ClassNotFoundException: jakarta.json.spi.JsonProvider
at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[na:1.8.0_181]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_181]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) ~[na:1.8.0_181]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_181]
... 66 common frames omitted
报错原因: jar包版本冲突，见下图
通过依赖树分析，可以看到jakarta.json-api存在版本冲突。
解决方式: 排除掉spring-data-elasticsearch中的对应包，在外面重新引入2.0.1版本的包。
2. 找不到类ToXContentObject
报错信息:
Class file for org.elasticsearch.common.xcontent.ToXContentObject not found
报错原因: 缺少jar包
解决方式: 引入elasticsearch-x-content包，具体见上文pom文件
3. ssl连接es异常
报错信息:
控制台报错：
java.net.SocketException: Network is unreachable: no further information
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[na:1.8.0_181]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) ~[na:1.8.0_181]
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processEvent(DefaultConnectingIOReactor.java:174) ~[httpcore-nio-4.4.16.jar:4.4.16]
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processEvents(DefaultConnectingIOReactor.java:148) ~[httpcore-nio-4.4.16.jar:4.4.16]
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:351) ~[httpcore-nio-4.4.16.jar:4.4.16]
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:221) ~[httpasyncclient-4.1.5.jar:4.1.5]
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64) ~[httpasyncclient-4.1.5.jar:4.1.5]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181]
网页端报错:
Caused by: javax.net.ssl.SSLHandshakeException: General SSLEngine problem
at sun.security.ssl.Handshaker.checkThrown(Handshaker.java:1529)
at sun.security.ssl.SSLEngineImpl.checkTaskThrown(SSLEngineImpl.java:535)
at sun.security.ssl.SSLEngineImpl.writeAppRecord(SSLEngineImpl.java:1214)
at sun.security.ssl.SSLEngineImpl.wrap(SSLEngineImpl.java:1186)
at javax.net.ssl.SSLEngine.wrap(SSLEngine.java:469)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doWrap(SSLIOSession.java:271)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doHandshake(SSLIOSession.java:317)
at org.apache.http.nio.reactor.ssl.SSLIOSession.isAppInputReady(SSLIOSession.java:545)
at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:120)
at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)
at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)
at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
... 1 more
Caused by: javax.net.ssl.SSLHandshakeException: General SSLEngine problem
at sun.security.ssl.Alerts.getSSLException(Alerts.java:192)
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1728)
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:330)
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:322)
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1614)
at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216)
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1052)
at sun.security.ssl.Handshaker$1.run(Handshaker.java:992)
at sun.security.ssl.Handshaker$1.run(Handshaker.java:989)
at java.security.AccessController.doPrivileged(Native Method)
at sun.security.ssl.Handshaker$DelegatedTask.run(Handshaker.java:1467)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doRunTask(SSLIOSession.java:289)
at org.apache.http.nio.reactor.ssl.SSLIOSession.doHandshake(SSLIOSession.java:357)
... 9 more
Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:397)
at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:302)
at sun.security.validator.Validator.validate(Validator.java:262)
at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324)
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:281)
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:136)
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1601)
... 17 more
报错原因: 连接es校验SSL证书失败
解决方式: 通过配置，跳过ssl证书校验
- 引入jar包
<dependency>
<groupId>io.github.hakky54groupId>
<artifactId>sslcontext-kickstartartifactId>
<version>7.1.0version>
dependency>
- 修改es连接配置
@Configuration
@ConditionalOnProperty(prefix = "es.client", name = "enabled", havingValue = "true")
public class MyClientConfig extends ElasticsearchConfiguration {
@Autowired
private EsClientProperties esClientProperties;
@Override
public ClientConfiguration clientConfiguration() {
//FIX 修改SSL证书校验失败的问题
SSLFactory sslFactory = SSLFactory.builder()
.withUnsafeTrustMaterial()
.withUnsafeHostnameVerifier()
.build();
return ClientConfiguration.builder()
.connectedTo(esClientProperties.getHost(), esClientProperties.getPort())
.usingSsl(sslFactory.getSslContext(), sslFactory.getHostnameVerifier())
.withBasicAuth(esClientProperties.getUsername(), esClientProperties.getPassword())
.build();
}
}
4. elastic用户无法授权
报错信息:
There was an unexpected error (type=Internal Server Error, status=500).
[es/get] failed: [security_exception] unable to authenticate user [elastic] for REST request [/spiderflow_news/_doc/c1281a144535]; nested exception is co.elastic.clients.elasticsearch._types.ElasticsearchException: [es/get] failed: [security_exception] unable to authenticate user [elastic] for REST request [/spiderflow_news/_doc/c1281a144535]
报错原因: 尚不明确
解决方式: 通过kibana页面重新添加用户，修改相关配置(主要是配置文件里的用户名和密码)。
以上是我在开发过程中遇到的问题及解决方式，希望对大家能有所帮助，有任何问题欢迎留言或者私信。
另：上文中所有提供的代码都是解决过已知问题后的正确代码，可以直接复用到项目或者工程中。