springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient
es客户端
es下载
es7.6.2版本下载地址
https://www.elastic.co/cn/downloads/past-releases#elasticsearch
我用的windows系统,下载
安装
解压后
完整配置
进入config目录,修改elasticsearch.yml配置文件,修改以下几个属性即可
-
cluster.name集群名称
-
node.name节点名称
-
network.host: 0.0.0.0 #设置外部ip可访问
-
http.port: 9200 #访问端口
-
cluster.initial_master_nodes
他的含义:你可以通过为 cluster.initial_master_nodes 参数设置一系列符合主节点条件的节点的主机名或 IP 地址来引导启动集群。你可以在命令行或 elasticsearch.yml 中提供这些信息。你还需要配置发现子系统，这样节点就知道如何找到彼此。
如果未设置 initial_master_nodes，那么在启动新节点时会尝试发现已有的集群。如果节点找不到可以加入的集群，将定期记录警告消息。 -
http.cors.enabled: true
http.cors.allow-origin: “*”
设置允许跨域 -
node.master: true # 默认为true,该节点是否有成为主节点的资格
node.data: true # 该节点是否存储数据
cluster.name: ql-application
node.name: ql-1
network.host: 0.0.0.0
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
如果后面控制台输出是中文乱码 ,可以在jvm.options中添加配置:
-Dfile.encoding=GBK
设置ik分词器
如果不了解为什么使用ik分词器,请自行了解
- 下载地址
https://github.com/medcl/elasticsearch-analysis-ik/releases
- 下载完成后,将其解压至elasticsearch的plugins,在plugins下创建一个ik文件夹,放在ik里面
启动es,查看是否成功
进入bin目录,启动elasticsearch.bat
如上图,则ik分词器插件添加成功,
浏览器输入: http://localhost:9200/?pretty
出现如下图,则为成功
一般,我们还会使用elasticsearch-head的插件来观察es客户端
建议使用谷歌插件elasticsearch-head
这里是我已经添加过索引word
pom文件
spring-data-elasticsearch版本
<dependency>
<groupId>org.springframework.datagroupId>
<artifactId>spring-data-elasticsearchartifactId>
<version>4.1.6version>
dependency>
springboot版本
<parent>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-parentartifactId>
<version>2.3.5.RELEASEversion>
<relativePath/>
parent>
因为目前spring-data-elasticsearch最高版本4.1.6对应的es版本是7.6.2,所以es客户端版本用7.6.2,否则springboot启动时会出现警告!
springboot集成es
yml文件配置
#elasticSearch设置
ql:
es:
esUrl: http:127.0.0.1:9200 #es节点,如果是多个节点用英文逗号隔开
connect-enabled: true #是否开启es连接
username: elastic #用户名
password: bych1234 #密码
user-enabled: false #是否开启用户登陆
connect-timeOut: 1000 #超时时间
socket-timeOut: 30000 #超时时间
connection-requestTimeOut: 500 #获取链接超时时间
max-connectNum: 100 #最大连接数
max-connectPerRoute: 100 #最大路由连接数
springboot启动配置
静态属性获取
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @author qinlei
* @description todo
* @date 2021/3/2 10:32
*/
@Component
@ConfigurationProperties(prefix = "ql.es",ignoreInvalidFields=true)
public class QLElasticsearchValue {
private String esUrl; //url
private boolean connectEnabled;
private String username;
private String password;
private boolean userEnabled;
private int connectTimeOut = 1000; // 连接超时时间
private int socketTimeOut = 30000; // 连接超时时间
private int connectionRequestTimeOut = 500; // 获取连接的超时时间
private int maxConnectNum = 100; // 最大连接数
private int maxConnectPerRoute = 100; // 最大路由连接数
//getter setter...
}
Configuration
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @author qinlei
* @description todo
* @date 2021/3/2 10:08
*/
@Configuration
@Slf4j
public class QLElasticsearchTemplate {
@Autowired
QLElasticsearchValue elasticsearchValue;
@Bean
RestHighLevelClient configRestHighLevelClient() {
if (!elasticsearchValue.isConnectEnabled()) {
log.info("==== ElasticSearch 未配置。 ====");
return null;
}
String[] esUrlArr = elasticsearchValue.getEsUrl().split(",");
List<HttpHost> httpHosts = new ArrayList<>();
for (String es : esUrlArr) {
String[] esUrlPort = es.split(":");
httpHosts.add(new HttpHost(esUrlPort[1], Integer.parseInt(esUrlPort[2]), esUrlPort[0]));
}
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
//需要用户名和密码的认证
if (elasticsearchValue.isUserEnabled()) {
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(elasticsearchValue.getUsername(), elasticsearchValue.getPassword()));
}
RestClientBuilder restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0]));
//异步连接httpclient配置
restClientBuilder.setHttpClientConfigCallback(
httpAsyncClientBuilder -> {
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
httpAsyncClientBuilder.setMaxConnTotal(elasticsearchValue.getMaxConnectNum());
httpAsyncClientBuilder.setMaxConnPerRoute(elasticsearchValue.getMaxConnectPerRoute());
return httpAsyncClientBuilder;
});
//异步连接httpclient延时配置
restClientBuilder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(elasticsearchValue.getConnectTimeOut());
requestConfigBuilder.setSocketTimeout(elasticsearchValue.getSocketTimeOut());
requestConfigBuilder.setConnectionRequestTimeout(elasticsearchValue.getConnectionRequestTimeOut());
return requestConfigBuilder;
});
//设置一个监听程序，每次节点发生故障时都会收到通知，这样就可以采取相应的措施。
restClientBuilder.setFailureListener(new RestClient.FailureListener() {
public void onFailure(HttpHost host) {
log.warn("==== ElasticSearch 服务错误");
log.warn("==== FAIL NODE : {}", host.toHostString());
}
});
log.info("ElasticSearch 服务连接成功。");
return new RestHighLevelClient(restClientBuilder);
}
}
相关操作类及模仿springdata的注解
注解
分词器配置
/**
* @author qinlei
* @description ik分词器
* @date 2021/3/19 10:48
*/
public enum AnalyzerType {
NO("不使用分词"),
/**
* 标准分词，默认分词器
*/
STANDARD("standard"),
/**
* ik_smart：会做最粗粒度的拆分；已被分出的词语将不会再次被其它词语占有
*/
IK_SMART("ik_smart"),
/**
* ik_max_word ：会将文本做最细粒度的拆分；尽可能多的拆分出词语
*/
IK_MAX_WORD("ik_max_word")
;
private String type;
AnalyzerType(String type){
this.type = type;
}
public String getType() {
return type;
}
}
字段属性
/**
* @author qinlei
* @description todo
* @date 2021/3/19 10:48
*/
public enum FieldType {
/**
* text
*/
TEXT("text"),
KEYWORD("keyword"),
INTEGER("integer"),
DOUBLE("double"),
DATE("date"),
/**
* 单条数据
*/
OBJECT("object"),
/**
* 嵌套数组
*/
NESTED("nested"),
;
FieldType(String type){
this.type = type;
}
private String type;
public String getType() {
return type;
}
}
document
在低版本的es客户端其实这里还有个type属性,但是在高版本已经弃用
import java.lang.annotation.*;
/**
* @author qinlei
* @description todo
* @date 2021/3/20 16:50
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
@Documented
@Inherited
public @interface Document {
/**
* index : 索引名称
* @return
*/
String indexName();
}
es文档的主键id
import java.lang.annotation.*;
/**
* @author qinlei
* @description todo
* @date 2021/3/20 16:51
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
@Documented
@Inherited
public @interface EsDataId {
}
doc字段注解
import org.jeecg.common.constant.enums.AnalyzerType;
import org.jeecg.common.constant.enums.FieldType;
import java.lang.annotation.*;
/**
* @author qinlei
* @description todo
* @date 2021/3/20 16:51
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
@Documented
@Inherited
public @interface Field {
FieldType type() default FieldType.TEXT;
/**
* 指定分词器
* @return
*/
AnalyzerType analyzer() default AnalyzerType.STANDARD;
}
es操作util,在这里面处理了注解
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageInfo;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import **.common.aspect.annotation.es.Document;
import **.common.aspect.annotation.es.EsDataId;
import **.common.constant.enums.FieldType;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;
/**
* @author qinlei
* @description es工具
* @date 2021/3/5 15:07
*/
@Slf4j
@Component
public class ESUtil {
@Resource
private RestHighLevelClient restHighLevelClient;
private static int index_number_of_shards = 5;//默认分片数
private static int index_number_of_replicas = 0;//默认副本数 单节点
public void setIndexNumber(int index_number_of_shards, int index_number_of_replicas) {
this.index_number_of_shards = index_number_of_shards;
this.index_number_of_replicas = index_number_of_replicas;
}
public RestHighLevelClient getInstance() {
return restHighLevelClient;
}
/**
* 创建索引(默认分片数为5和副本数为1)
*
* @param clazz 根据实体自动映射es索引
* @throws IOException
*/
public boolean createIndex(Class clazz) throws Exception {
Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
}
String indexName = declaredAnnotation.indexName();
boolean flag = createRootIndex(indexName, clazz);
if (flag) {
log.info("创建索引成功！索引名称为{}", indexName);
return true;
}
return false;
}
/**
* 创建索引(默认分片数为5和副本数为1)
*
* @param clazz 根据实体自动映射es索引
* @throws IOException
*/
public boolean createIndexIfNotExist(Class clazz) throws Exception {
Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
}
String indexName = declaredAnnotation.indexName();
boolean indexExists = isIndexExists(indexName);
if (!indexExists) {
boolean flag = createRootIndex(indexName, clazz);
if (flag) {
log.info("创建索引成功！索引名称为{}", indexName);
return true;
}
}
return false;
}
private boolean createRootIndex(String indexName, Class clazz) throws IOException {
CreateIndexRequest request = new CreateIndexRequest(indexName);
request.settings(Settings.builder()
// 设置分片数， 副本数
.put("index.number_of_shards", index_number_of_shards)
.put("index.number_of_replicas", index_number_of_replicas)
);
request.mapping(generateBuilder(clazz));
CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
// 指示是否所有节点都已确认请求
boolean acknowledged = response.isAcknowledged();
// 指示是否在超时之前为索引中的每个分片启动了必需的分片副本数
boolean shardsAcknowledged = response.isShardsAcknowledged();
return acknowledged || shardsAcknowledged;
}
/**
* 更新索引(默认分片数为5和副本数为1)：
* 只能给索引上添加一些不存在的字段
* 已经存在的映射不能改
*
* @param clazz 根据实体自动映射es索引
* @throws IOException
*/
public boolean updateIndex(Class clazz) throws Exception {
Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
}
String indexName = declaredAnnotation.indexName();
PutMappingRequest request = new PutMappingRequest(indexName);
request.source(generateBuilder(clazz));
AcknowledgedResponse response = restHighLevelClient.indices().putMapping(request, RequestOptions.DEFAULT);
// 指示是否所有节点都已确认请求
boolean acknowledged = response.isAcknowledged();
if (acknowledged) {
log.info("更新索引索引成功！索引名称为{}", indexName);
return true;
}
return false;
}
/**
* 删除索引
*
* @param indexName
* @return
*/
public boolean delIndex(String indexName) {
boolean acknowledged = false;
try {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName);
deleteIndexRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
AcknowledgedResponse delete = restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
acknowledged = delete.isAcknowledged();
} catch (IOException e) {
e.printStackTrace();
}
return acknowledged;
}
/**
* 判断索引是否存在
*
* @param indexName
* @return
*/
public boolean isIndexExists(String indexName) {
boolean exists = false;
try {
GetIndexRequest getIndexRequest = new GetIndexRequest(indexName);
getIndexRequest.humanReadable(true);
exists = restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return exists;
}
/**
* 添加单条数据
* 提供多种方式：
* 1. json
* 2. map
* Map jsonMap = new HashMap<>();
* jsonMap.put("user", "kimchy");
* jsonMap.put("postDate", new Date());
* jsonMap.put("message", "trying out Elasticsearch");
* IndexRequest indexRequest = new IndexRequest("posts")
* .id("1").source(jsonMap);
* 3. builder
* XContentBuilder builder = XContentFactory.jsonBuilder();
* builder.startObject();
* {
* builder.field("user", "kimchy");
* builder.timeField("postDate", new Date());
* builder.field("message", "trying out Elasticsearch");
* }
* builder.endObject();
* IndexRequest indexRequest = new IndexRequest("posts")
* .id("1").source(builder);
* 4. source:
* IndexRequest indexRequest = new IndexRequest("posts")
* .id("1")
* .source("user", "kimchy",
* "postDate", new Date(),
* "message", "trying out Elasticsearch");
*
* 报错： Validation Failed: 1: type is missing;
* 加入两个jar包解决
*
* 提供新增或修改的功能
*
* @return
*/
public IndexResponse index(Object o) throws Exception {
Document declaredAnnotation = (Document) o.getClass().getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", o.getClass().getName()));
}
String indexName = declaredAnnotation.indexName();
IndexRequest request = new IndexRequest(indexName);
Field fieldByAnnotation = getFieldByAnnotation(o, EsDataId.class);
if (fieldByAnnotation != null) {
fieldByAnnotation.setAccessible(true);
try {
Object id = fieldByAnnotation.get(o);
request = request.id(id.toString());
} catch (IllegalAccessException e) {
log.error("获取id字段出错：{}", e);
}
}
String userJson = JSON.toJSONString(o);
request.source(userJson, XContentType.JSON);
IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
return indexResponse;
}
/**
* 根据id查询
*
* @return
*/
public String queryById(String indexName, String id) throws IOException {
GetRequest getRequest = new GetRequest(indexName, id);
// getRequest.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE);
GetResponse getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
String jsonStr = getResponse.getSourceAsString();
return jsonStr;
}
/**
* 查询封装返回json字符串
*
* @param indexName
* @param searchSourceBuilder
* @return
* @throws IOException
*/
public String search(String indexName, SearchSourceBuilder searchSourceBuilder) throws IOException {
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
searchRequest.scroll(TimeValue.timeValueMinutes(1L));
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHits hits = searchResponse.getHits();
JSONArray jsonArray = new JSONArray();
for (SearchHit hit : hits) {
String sourceAsString = hit.getSourceAsString();
JSONObject jsonObject = JSON.parseObject(sourceAsString);
jsonArray.add(jsonObject);
}
log.info("返回总数为：" + hits.getTotalHits());
return jsonArray.toJSONString();
}
/**
* 查询封装，带分页
*
* @param searchSourceBuilder
* @param pageNum
* @param pageSize
* @param s
* @param
* @return
* @throws IOException
*/
public <T> PageInfo<T> search(SearchSourceBuilder searchSourceBuilder, int pageNum, int pageSize, Class<T> s) throws Exception {
Document declaredAnnotation = (Document) s.getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName()));
}
String indexName = declaredAnnotation.indexName();
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
JSONArray jsonArray = new JSONArray();
for (SearchHit hit : hits) {
String sourceAsString = hit.getSourceAsString();
JSONObject jsonObject = JSON.parseObject(sourceAsString);
jsonArray.add(jsonObject);
}
log.info("返回总数为：" + hits.getTotalHits());
int total = (int) hits.getTotalHits().value;
// 封装分页
List<T> list = jsonArray.toJavaList(s);
PageInfo<T> page = new PageInfo<>();
page.setList(list);
page.setPageNum(pageNum);
page.setPageSize(pageSize);
page.setTotal(total);
page.setPages(total == 0 ? 0 : (total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1));
page.setHasNextPage(page.getPageNum() < page.getPages());
return page;
}
/**
* 查询封装，返回集合
*
* @param searchSourceBuilder
* @param s
* @param
* @return
* @throws IOException
*/
public <T> List<T> search(SearchSourceBuilder searchSourceBuilder, Class<T> s) throws Exception {
Document declaredAnnotation = (Document) s.getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName()));
}
String indexName = declaredAnnotation.indexName();
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
searchRequest.scroll(TimeValue.timeValueMinutes(1L));
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// //配置标题高亮显示
// HighlightBuilder highlightBuilder = new HighlightBuilder(); //生成高亮查询器
// highlightBuilder.field(title); //高亮查询字段
// highlightBuilder.field(content); //高亮查询字段
// highlightBuilder.requireFieldMatch(false); //如果要多个字段高亮,这项要为false
// highlightBuilder.preTags(""); //高亮设置
// highlightBuilder.postTags("");
//
// //下面这两项,如果你要高亮如文字内容等有很多字的字段,必须配置,不然会导致高亮不全,文章内容缺失等
// highlightBuilder.fragmentSize(800000); //最大高亮分片数
// highlightBuilder.numOfFragments(0); //从第一个分片获取高亮片段
String scrollId = searchResponse.getScrollId();
SearchHits hits = searchResponse.getHits();
JSONArray jsonArray = new JSONArray();
for (SearchHit hit : hits) {
String sourceAsString = hit.getSourceAsString();
JSONObject jsonObject = JSON.parseObject(sourceAsString);
jsonArray.add(jsonObject);
}
// 封装分页
List<T> list = jsonArray.toJavaList(s);
return list;
}
/**
* 批量插入文档
* 文档存在 则插入
* 文档不存在 则更新
*
* @param list
* @return
*/
public <T> boolean batchSaveOrUpdate(List<T> list, boolean izAsync) throws Exception {
Object o1 = list.get(0);
Document declaredAnnotation = (Document) o1.getClass().getDeclaredAnnotation(Document.class);
if (declaredAnnotation == null) {
throw new Exception(String.format("class name: %s can not find Annotation [@Document], please check", o1.getClass().getName()));
}
String indexName = declaredAnnotation.indexName();
BulkRequest request = new BulkRequest(indexName);
for (Object o : list) {
String jsonStr = JSON.toJSONString(o);
IndexRequest indexReq = new IndexRequest().source(jsonStr, XContentType.JSON);
Field fieldByAnnotation = getFieldByAnnotation(o, EsDataId.class);
if (fieldByAnnotation != null) {
fieldByAnnotation.setAccessible(true);
try {
Object id = fieldByAnnotation.get(o);
indexReq = indexReq.id(id.toString());
} catch (IllegalAccessException e) {
log.error("获取id字段出错：{}", e);
}
}
request.add(indexReq);
}
if (izAsync) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
return outResult(bulkResponse);
} else {
restHighLevelClient.bulkAsync(request, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkResponse) {
outResult(bulkResponse);
}
@Override
public void onFailure(Exception e) {
log.error("es数据 添加错误{}", e);
}
});
}
return true;
}
private boolean outResult(BulkResponse bulkResponse) {
for (BulkItemResponse bulkItemResponse : bulkResponse) {
DocWriteResponse itemResponse = bulkItemResponse.getResponse();
IndexResponse indexResponse = (IndexResponse) itemResponse;
log.info("单条返回结果：{}", indexResponse);
if (bulkItemResponse.isFailed()) {
log.error("es 返回错误{}", bulkItemResponse.getFailureMessage());
return false;
}
}
return true;
}
/**
* 删除文档
*
* @param indexName： 索引名称
* @param docId： 文档id
*/
public boolean deleteDoc(String indexName, String docId) throws IOException {
DeleteRequest request = new DeleteRequest(indexName, docId);
DeleteResponse deleteResponse = restHighLevelClient.delete(request, RequestOptions.DEFAULT);
// 解析response
String index = deleteResponse.getIndex();
String id = deleteResponse.getId();
long version = deleteResponse.getVersion();
ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure :
shardInfo.getFailures()) {
String reason = failure.reason();
log.info("删除失败，原因为 {}", reason);
}
}
return true;
}
/**
* 根据json类型更新文档
*
* @param indexName
* @param docId
* @param o
* @return
* @throws IOException
*/
public boolean updateDoc(String indexName, String docId, Object o) throws IOException {
UpdateRequest request = new UpdateRequest(indexName, docId);
request.doc(JSON.toJSONString(o), XContentType.JSON);
UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT);
String index = updateResponse.getIndex();
String id = updateResponse.getId();
long version = updateResponse.getVersion();
if (updateResponse.getResult() == DocWriteResponse.Result.CREATED) {
return true;
} else if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
} else if (updateResponse.getResult() == DocWriteResponse.Result.DELETED) {
} else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) {
}
return false;
}
/**
* 根据Map类型更新文档
*
* @param indexName
* @param docId
* @param map
* @return
* @throws IOException
*/
public boolean updateDoc(String indexName, String docId, Map<String, Object> map) throws IOException {
UpdateRequest request = new UpdateRequest(indexName, docId);
request.doc(map);
UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT);
String index = updateResponse.getIndex();
String id = updateResponse.getId();
long version = updateResponse.getVersion();
if (updateResponse.getResult() == DocWriteResponse.Result.CREATED) {
return true;
} else if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
} else if (updateResponse.getResult() == DocWriteResponse.Result.DELETED) {
} else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) {
}
return false;
}
public XContentBuilder generateBuilder(Class clazz) throws IOException {
// 获取索引名称及类型
Document doc = (Document) clazz.getAnnotation(Document.class);
System.out.println(doc.indexName());
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.startObject("properties");
Field[] declaredFields = clazz.getDeclaredFields();
for (Field f : declaredFields) {
if (f.isAnnotationPresent(org.jeecg.common.aspect.annotation.es.Field.class)) {
// 获取注解
org.jeecg.common.aspect.annotation.es.Field declaredAnnotation =
f.getDeclaredAnnotation(org.jeecg.common.aspect.annotation.es.Field.class);
// 如果嵌套对象：
/**
* {
* "mappings": {
* "properties": {
* "region": {
* "type": "keyword"
* },
* "manager": {
* "properties": {
* "age": { "type": "integer" },
* "name": {
* "properties": {
* "first": { "type": "text" },
* "last": { "type": "text" }
* }
* }
* }
* }
* }
* }
* }
*/
if (declaredAnnotation.type() == FieldType.OBJECT) {
// 获取当前类的对象-- Action
Class<?> type = f.getType();
Field[] df2 = type.getDeclaredFields();
builder.startObject(f.getName());
builder.startObject("properties");
// 遍历该对象中的所有属性
for (Field f2 : df2) {
if (f2.isAnnotationPresent(org.jeecg.common.aspect.annotation.es.Field.class)) {
// 获取注解
org.jeecg.common.aspect.annotation.es.Field declaredAnnotation2 = f2.getDeclaredAnnotation(org.jeecg.common.aspect.annotation.es.Field.class);
builder.startObject(f2.getName());
builder.field("type", declaredAnnotation2.type().getType());
// keyword不需要分词
if (declaredAnnotation2.type() == FieldType.TEXT) {
builder.field("analyzer", declaredAnnotation2.analyzer().getType());
}
if (declaredAnnotation2.type() == FieldType.DATE) {
builder.field("format", "yyyy-MM-dd HH:mm:ss");
}
builder.endObject();
}
}
builder.endObject();
builder.endObject();
} else {
builder.startObject(f.getName());
builder.field("type", declaredAnnotation.type().getType());
// keyword不需要分词
if (declaredAnnotation.type() == FieldType.TEXT) {
builder.field("analyzer", declaredAnnotation.analyzer().getType());
}
if (declaredAnnotation.type() == FieldType.DATE) {
builder.field("format", "yyyy-MM-dd HH:mm:ss");
}
builder.endObject();
}
}
}
// 对应property
builder.endObject();
builder.endObject();
return builder;
}
public static Field getFieldByAnnotation(Object o, Class annotationClass) {
Field[] declaredFields = o.getClass().getDeclaredFields();
if (declaredFields != null && declaredFields.length > 0) {
for (Field f : declaredFields) {
if (f.isAnnotationPresent(annotationClass)) {
return f;
}
}
}
return null;
}
/**
* 获取低水平客户端
*
* @return
*/
public RestClient getLowLevelClient() {
return restHighLevelClient.getLowLevelClient();
}
/**
* 高亮结果集 特殊处理
* map转对象 JSONObject.parseObject(JSONObject.toJSONString(map), Content.class)
*
* @param searchResponse
* @param highlightField
*/
public List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) {
//解析结果
ArrayList<Map<String, Object>> list = new ArrayList<>();
for (SearchHit hit : searchResponse.getHits().getHits()) {
Map<String, HighlightField> high = hit.getHighlightFields();
HighlightField title = high.get(highlightField);
hit.getSourceAsMap().put("id", hit.getId());
Map<String, Object> sourceAsMap = hit.getSourceAsMap();//原来的结果
//解析高亮字段,将原来的字段换为高亮字段
if (title != null) {
Text[] texts = title.fragments();
String nTitle = "";
for (Text text : texts) {
nTitle += text;
}
//替换
sourceAsMap.put(highlightField, nTitle);
}
list.add(sourceAsMap);
}
return list;
}
/**
* 查询并分页
*
* @param index 索引名称
* @param query 查询条件
* @param size 文档大小限制
* @param from 从第几页开始
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @param sortField 排序字段
* @param highlightField 高亮字段
* @return
*/
public List<Map<String, Object>> searchListData(String index,
SearchSourceBuilder query,
Integer size,
Integer from,
String fields,
String sortField,
String highlightField) throws IOException {
SearchRequest request = new SearchRequest(index);
SearchSourceBuilder builder = query;
if (StringUtils.isNotEmpty(fields)) {
//只查询特定字段。如果需要查询所有字段则不设置该项。
builder.fetchSource(new FetchSourceContext(true, fields.split(","), Strings.EMPTY_ARRAY));
}
from = from <= 0 ? 0 : from * size;
//设置确定结果要从哪个索引开始搜索的from选项，默认为0
builder.from(from);
builder.size(size);
if (StringUtils.isNotEmpty(sortField)) {
//排序字段，注意如果proposal_no是text类型会默认带有keyword性质，需要拼接.keyword
builder.sort(sortField + ".keyword", SortOrder.ASC);
}
//高亮
HighlightBuilder highlight = new HighlightBuilder();
highlight.field(highlightField);
//关闭多个高亮
highlight.requireFieldMatch(false);
highlight.preTags("");
highlight.postTags("");
builder.highlighter(highlight);
//不返回源数据。只有条数之类的数据。
//builder.fetchSource(false);
request.source(builder);
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
log.error("==" + response.getHits().getTotalHits());
if (response.status().getStatus() == 200) {
// 解析对象
return setSearchResponse(response, highlightField);
}
return null;
}
}
测试
pojo
import org.jeecg.common.aspect.annotation.es.Document;
import org.jeecg.common.aspect.annotation.es.EsDataId;
import org.jeecg.common.aspect.annotation.es.Field;
import org.jeecg.common.constant.enums.AnalyzerType;
import org.jeecg.common.constant.enums.FieldType;
import java.io.Serializable;
import java.util.Objects;
/**
* @author qinlei
* @description todo
* @date 2021/3/20 14:17
*/
@Document(indexName = "word")
public class DocumentWord implements Serializable {
private static final long serialVersionUID = 8272130486926494273L;
@EsDataId
@Field(type = FieldType.TEXT)
private String id;
@Field(type = FieldType.TEXT,analyzer = AnalyzerType.IK_MAX_WORD)
private String title;
@Field(type = FieldType.TEXT,analyzer = AnalyzerType.IK_SMART)
private String content;
public DocumentWord() {
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof DocumentWord)) return false;
DocumentWord that = (DocumentWord) o;
return Objects.equals(getId(), that.getId()) && Objects.equals(getTitle(), that.getTitle()) && Objects.equals(getContent(), that.getContent());
}
@Override
public int hashCode() {
return Objects.hash(getId(), getTitle(), getContent());
}
@Override
public String toString() {
return "DocumentWord{" +
"id='" + id + '\'' +
", title='" + title + '\'' +
", content='" + content + '\'' +
'}';
}
}
service
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.formula.functions.T;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jeecg.common.aspect.annotation.es.Document;
import org.jeecg.common.util.ESUtil;
import org.jeecg.modules.demo.es.pojo.DocumentWord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
/**
* @author qinlei
* @description todo
* @date 2021/3/20 14:42
*/
@Service
@Slf4j
public class EsImpl<T> {
@Autowired
private ESUtil esUtil;
public boolean removeIndex(String indexName){
return esUtil.delIndex(indexName);
}
public boolean addData(List<T> list) {
if(list.size()<0){
return true;
}
try {
if(esUtil.createIndex(list.get(0).getClass()))
return esUtil.batchSaveOrUpdate(list,true);
else
return false;
} catch (Exception e) {
log.error("############### 数据添加失败! {}", e);
}
return false;
}
public List<DocumentWord> search(SearchSourceBuilder searchSourceBuilder) {
try {
return esUtil.search(searchSourceBuilder, DocumentWord.class);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
controller
package org.jeecg.modules.demo.es.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.demo.es.pojo.DocumentWord;
import org.jeecg.modules.demo.es.service.EsImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author qinlei
* @description todo
* @date 2021/3/5 16:31
*/
@Api("es测试")
@RestController
@RequestMapping(("/es"))
public class SearchController {
@Autowired
private EsImpl<DocumentWord> searchServiceImp;
@ApiOperation("删除索引")
@PostMapping("/removeIndex/{indexName}")
public Result removeIndex(@PathVariable("indexName") String indexName){
return Result.OK(searchServiceImp.removeIndex(indexName));
}
@ApiOperation("新增数据")
@PostMapping("addData")
public Result addData(HttpServletRequest request){
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
MultipartFile multipartFile = multipartRequest.getFile("file");
InputStreamReader isr = null;
BufferedReader bReader = null;
StringBuilder sb = new StringBuilder();
try {
isr = new InputStreamReader(multipartFile.getInputStream());
bReader = new BufferedReader(isr);
String s = "";
while ((s =bReader.readLine()) != null) {
sb.append(s + "\n");
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
if(bReader != null){
bReader.close();
}
if(isr != null){
isr.close();
}
} catch (IOException ioException) {
ioException.printStackTrace();
}
}
DocumentWord documentWord = new DocumentWord();
documentWord.setId("1");
documentWord.setTitle("******手册");
documentWord.setContent(sb.toString());
List<DocumentWord> list = new ArrayList<>();
list.add(documentWord);
return Result.OK(searchServiceImp.addData(list));
}
@ApiOperation("查询数据")
@GetMapping("queryDataById")
public Result queryDataById(@RequestParam("searchType") String searchType,
@RequestParam("searchValue") String searchValue){
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.from(0);
searchSourceBuilder.size(5);
// 符合条件查询
BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
boolBuilder.must(QueryBuilders.termQuery(searchType, searchValue));
searchSourceBuilder.query(boolBuilder);
return Result.OK(searchServiceImp.search(searchSourceBuilder));
}
}
副本小坑理解
因为我这里只有一个节点,所以设置副本为0,当设置>0的时候,es健康状态为黄色
比如设置副本为1,分片为4,es实际分配时,当前节点分配2个分片,还有两个会寻找其他节点,由于节点只有一个,找不到其他的node,所以会报黄色健康
ik分词器的两个属性
ik_max_word:
会将文本做最细粒度的拆分；尽可能多的拆分出词语
用他其实就是可以进行最大程度的模糊查询
ik_smart:
会做最粗粒度的拆分；已被分出的词语将不会再次被其它词语占有
用它其实就是进行精确查找
参考
工具类及注解部分copy(做了小部分优化):
https://blog.csdn.net/lsqingfeng/article/details/107620558
其实核心有点类似模仿springdataelasticsearch封装