springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient

es客户端

es下载

es7.6.2版本下载地址

https://www.elastic.co/cn/downloads/past-releases#elasticsearch

我用的windows系统,下载
springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第1张图片

安装

解压后
springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第2张图片

完整配置

进入config目录,修改elasticsearch.yml配置文件,修改以下几个属性即可

  1. cluster.name集群名称

  2. node.name节点名称

  3. network.host: 0.0.0.0 #设置外部ip可访问

  4. http.port: 9200 #访问端口

  5. cluster.initial_master_nodes
    他的含义:

    你可以通过为 cluster.initial_master_nodes 参数设置一系列符合主节点条件的节点的主机名或 IP 地址来引导启动集群。你可以在命令行或 elasticsearch.yml 中提供这些信息。你还需要配置发现子系统,这样节点就知道如何找到彼此。
    如果未设置 initial_master_nodes,那么在启动新节点时会尝试发现已有的集群。如果节点找不到可以加入的集群,将定期记录警告消息。

  6. http.cors.enabled: true
    http.cors.allow-origin: “*”
    设置允许跨域

  7. node.master: true # 默认为true,该节点是否有成为主节点的资格
    node.data: true # 该节点是否存储数据


cluster.name: ql-application

node.name: ql-1

network.host: 0.0.0.0

http.port: 9200

http.cors.enabled: true 
http.cors.allow-origin: "*"
node.master: true
node.data: true

如果后面控制台输出是中文乱码 ,可以在jvm.options中添加配置:

-Dfile.encoding=GBK

设置ik分词器

如果不了解为什么使用ik分词器,请自行了解

  1. 下载地址

https://github.com/medcl/elasticsearch-analysis-ik/releases

  1. 下载完成后,将其解压至elasticsearch的plugins,在plugins下创建一个ik文件夹,放在ik里面
    springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第3张图片

启动es,查看是否成功

进入bin目录,启动elasticsearch.bat

springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第4张图片
如上图,则ik分词器插件添加成功,
浏览器输入: http://localhost:9200/?pretty
出现如下图,则为成功
springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第5张图片
一般,我们还会使用elasticsearch-head的插件来观察es客户端

建议使用谷歌插件elasticsearch-head
springboot集成elasticSearch,使用spring-data-elasticsearch RestHighLevelClient_第6张图片
这里是我已经添加过索引word

pom文件

spring-data-elasticsearch版本

		
        <dependency>
            <groupId>org.springframework.datagroupId>
            <artifactId>spring-data-elasticsearchartifactId>
            <version>4.1.6version>
        dependency>
        

springboot版本

	<parent>
	    <groupId>org.springframework.bootgroupId>
	    <artifactId>spring-boot-starter-parentartifactId>
	    <version>2.3.5.RELEASEversion>
	    <relativePath/>
	parent>

因为目前spring-data-elasticsearch最高版本4.1.6对应的es版本是7.6.2,所以es客户端版本用7.6.2,否则springboot启动时会出现警告!

springboot集成es

yml文件配置

#elasticSearch设置
ql:
  es:
    esUrl: http:127.0.0.1:9200 #es节点,如果是多个节点用英文逗号隔开
    connect-enabled: true #是否开启es连接
    username: elastic	#用户名
    password: bych1234	#密码
    user-enabled: false	#是否开启用户登陆
    connect-timeOut: 1000 #超时时间
    socket-timeOut: 30000 #超时时间
    connection-requestTimeOut: 500 #获取链接超时时间
    max-connectNum: 100 #最大连接数
    max-connectPerRoute: 100 #最大路由连接数

springboot启动配置

静态属性获取


import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/2 10:32
 */
@Component
@ConfigurationProperties(prefix = "ql.es",ignoreInvalidFields=true)
public class QLElasticsearchValue {
    private String esUrl;   //url
    private boolean connectEnabled;
    private String username;
    private String password;
    private boolean userEnabled;
    private int connectTimeOut = 1000; // 连接超时时间
    private int socketTimeOut = 30000; // 连接超时时间
    private int connectionRequestTimeOut = 500; // 获取连接的超时时间
    private int maxConnectNum = 100; // 最大连接数
    private int maxConnectPerRoute = 100; // 最大路由连接数

    //getter setter...
}


Configuration


import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/2 10:08
 */
@Configuration
@Slf4j
public class QLElasticsearchTemplate {

    @Autowired
    QLElasticsearchValue elasticsearchValue;

    @Bean
    RestHighLevelClient configRestHighLevelClient() {
        if (!elasticsearchValue.isConnectEnabled()) {
            log.info("==== ElasticSearch 未配置。 ====");
            return null;
        }
        String[] esUrlArr = elasticsearchValue.getEsUrl().split(",");
        List<HttpHost> httpHosts = new ArrayList<>();
        for (String es : esUrlArr) {
            String[] esUrlPort = es.split(":");
            httpHosts.add(new HttpHost(esUrlPort[1], Integer.parseInt(esUrlPort[2]), esUrlPort[0]));
        }

        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        //需要用户名和密码的认证
        if (elasticsearchValue.isUserEnabled()) {
            credentialsProvider.setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(elasticsearchValue.getUsername(), elasticsearchValue.getPassword()));
        }
        RestClientBuilder restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0]));
        //异步连接httpclient配置
        restClientBuilder.setHttpClientConfigCallback(
                        httpAsyncClientBuilder -> {
                            httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                            httpAsyncClientBuilder.setMaxConnTotal(elasticsearchValue.getMaxConnectNum());
                            httpAsyncClientBuilder.setMaxConnPerRoute(elasticsearchValue.getMaxConnectPerRoute());
                            return httpAsyncClientBuilder;
                        });
        //异步连接httpclient延时配置
        restClientBuilder.setRequestConfigCallback(requestConfigBuilder -> {
                    requestConfigBuilder.setConnectTimeout(elasticsearchValue.getConnectTimeOut());
                    requestConfigBuilder.setSocketTimeout(elasticsearchValue.getSocketTimeOut());
                    requestConfigBuilder.setConnectionRequestTimeout(elasticsearchValue.getConnectionRequestTimeOut());
                    return requestConfigBuilder;
                });
        //设置一个监听程序,每次节点发生故障时都会收到通知,这样就可以采取相应的措施。
        restClientBuilder.setFailureListener(new RestClient.FailureListener() {
            public void onFailure(HttpHost host) {
                log.warn("==== ElasticSearch 服务错误");
                log.warn("==== FAIL NODE : {}", host.toHostString());
            }
        });
        log.info("ElasticSearch 服务连接成功。");
        return new RestHighLevelClient(restClientBuilder);
    }

}

相关操作类及模仿springdata的注解

注解

分词器配置


/**
 * @author qinlei
 * @description ik分词器
 * @date 2021/3/19 10:48
 */
public enum AnalyzerType {

    NO("不使用分词"),
    /**
     * 标准分词,默认分词器
     */
    STANDARD("standard"),

    /**
     * ik_smart:会做最粗粒度的拆分;已被分出的词语将不会再次被其它词语占有
     */
    IK_SMART("ik_smart"),

    /**
     * ik_max_word :会将文本做最细粒度的拆分;尽可能多的拆分出词语
     */
    IK_MAX_WORD("ik_max_word")

    ;

    private String type;

    AnalyzerType(String type){
        this.type = type;
    }

    public String getType() {
        return type;
    }
}

字段属性


/**
 * @author qinlei
 * @description todo
 * @date 2021/3/19 10:48
 */
public enum FieldType {
    /**
     * text
     */
    TEXT("text"),

    KEYWORD("keyword"),

    INTEGER("integer"),

    DOUBLE("double"),

    DATE("date"),

    /**
     * 单条数据
     */
    OBJECT("object"),

    /**
     * 嵌套数组
     */
    NESTED("nested"),


    ;


    FieldType(String type){
        this.type = type;
    }

    private String type;

    public String getType() {
        return type;
    }
}

document

在低版本的es客户端其实这里还有个type属性,但是在高版本已经弃用

import java.lang.annotation.*;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/20 16:50
 */
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
@Documented
@Inherited
public @interface Document {
    /**
     * index : 索引名称
     * @return
     */
    String indexName();
    
}

es文档的主键id


import java.lang.annotation.*;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/20 16:51
 */
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
@Documented
@Inherited
public @interface EsDataId {

}



doc字段注解


import org.jeecg.common.constant.enums.AnalyzerType;
import org.jeecg.common.constant.enums.FieldType;

import java.lang.annotation.*;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/20 16:51
 */
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
@Documented
@Inherited
public @interface Field {

    FieldType type() default FieldType.TEXT;

    /**
     * 指定分词器
     * @return
     */
    AnalyzerType analyzer() default AnalyzerType.STANDARD;
}

es操作util,在这里面处理了注解


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageInfo;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import **.common.aspect.annotation.es.Document;
import **.common.aspect.annotation.es.EsDataId;
import **.common.constant.enums.FieldType;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;

/**
 * @author qinlei
 * @description es工具
 * @date 2021/3/5 15:07
 */
@Slf4j
@Component
public class ESUtil {

    @Resource
    private RestHighLevelClient restHighLevelClient;

    private static int index_number_of_shards = 5;//默认分片数

    private static int index_number_of_replicas = 0;//默认副本数  单节点

    public void setIndexNumber(int index_number_of_shards, int index_number_of_replicas) {
        this.index_number_of_shards = index_number_of_shards;
        this.index_number_of_replicas = index_number_of_replicas;
    }

    public RestHighLevelClient getInstance() {
        return restHighLevelClient;
    }

    /**
     * 创建索引(默认分片数为5和副本数为1)
     *
     * @param clazz 根据实体自动映射es索引
     * @throws IOException
     */
    public boolean createIndex(Class clazz) throws Exception {
        Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
        if (declaredAnnotation == null) {
            throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
        }
        String indexName = declaredAnnotation.indexName();
        boolean flag = createRootIndex(indexName, clazz);
        if (flag) {
            log.info("创建索引成功!索引名称为{}", indexName);
            return true;
        }
        return false;
    }

    /**
     * 创建索引(默认分片数为5和副本数为1)
     *
     * @param clazz 根据实体自动映射es索引
     * @throws IOException
     */
    public boolean createIndexIfNotExist(Class clazz) throws Exception {
        Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
        if (declaredAnnotation == null) {
            throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
        }
        String indexName = declaredAnnotation.indexName();

        boolean indexExists = isIndexExists(indexName);
        if (!indexExists) {
            boolean flag = createRootIndex(indexName, clazz);
            if (flag) {
                log.info("创建索引成功!索引名称为{}", indexName);
                return true;
            }
        }
        return false;
    }

    private boolean createRootIndex(String indexName, Class clazz) throws IOException {
        CreateIndexRequest request = new CreateIndexRequest(indexName);
        request.settings(Settings.builder()
                // 设置分片数, 副本数
                .put("index.number_of_shards", index_number_of_shards)
                .put("index.number_of_replicas", index_number_of_replicas)
        );
        request.mapping(generateBuilder(clazz));
        CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
        // 指示是否所有节点都已确认请求
        boolean acknowledged = response.isAcknowledged();
        // 指示是否在超时之前为索引中的每个分片启动了必需的分片副本数
        boolean shardsAcknowledged = response.isShardsAcknowledged();
        return acknowledged || shardsAcknowledged;
    }

    /**
     * 更新索引(默认分片数为5和副本数为1):
     * 只能给索引上添加一些不存在的字段
     * 已经存在的映射不能改
     *
     * @param clazz 根据实体自动映射es索引
     * @throws IOException
     */
    public boolean updateIndex(Class clazz) throws Exception {
        Document declaredAnnotation = (Document) clazz.getDeclaredAnnotation(Document.class);
        if (declaredAnnotation == null) {
            throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", clazz.getName()));
        }
        String indexName = declaredAnnotation.indexName();
        PutMappingRequest request = new PutMappingRequest(indexName);

        request.source(generateBuilder(clazz));
        AcknowledgedResponse response = restHighLevelClient.indices().putMapping(request, RequestOptions.DEFAULT);
        // 指示是否所有节点都已确认请求
        boolean acknowledged = response.isAcknowledged();

        if (acknowledged) {
            log.info("更新索引索引成功!索引名称为{}", indexName);
            return true;
        }
        return false;
    }

    /**
     * 删除索引
     *
     * @param indexName
     * @return
     */
    public boolean delIndex(String indexName) {
        boolean acknowledged = false;
        try {
            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName);
            deleteIndexRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
            AcknowledgedResponse delete = restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
            acknowledged = delete.isAcknowledged();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return acknowledged;
    }

    /**
     * 判断索引是否存在
     *
     * @param indexName
     * @return
     */
    public boolean isIndexExists(String indexName) {
        boolean exists = false;
        try {
            GetIndexRequest getIndexRequest = new GetIndexRequest(indexName);
            getIndexRequest.humanReadable(true);
            exists = restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return exists;
    }


    /**
     * 添加单条数据
     * 提供多种方式:
     * 1. json
     * 2. map
     * Map jsonMap = new HashMap<>();
     * jsonMap.put("user", "kimchy");
     * jsonMap.put("postDate", new Date());
     * jsonMap.put("message", "trying out Elasticsearch");
     * IndexRequest indexRequest = new IndexRequest("posts")
     * .id("1").source(jsonMap);
     * 3. builder
     * XContentBuilder builder = XContentFactory.jsonBuilder();
     * builder.startObject();
     * {
     * builder.field("user", "kimchy");
     * builder.timeField("postDate", new Date());
     * builder.field("message", "trying out Elasticsearch");
     * }
     * builder.endObject();
     * IndexRequest indexRequest = new IndexRequest("posts")
     * .id("1").source(builder);
     * 4. source:
     * IndexRequest indexRequest = new IndexRequest("posts")
     * .id("1")
     * .source("user", "kimchy",
     * "postDate", new Date(),
     * "message", "trying out Elasticsearch");
     * 

* 报错: Validation Failed: 1: type is missing; * 加入两个jar包解决 *

* 提供新增或修改的功能 * * @return */ public IndexResponse index(Object o) throws Exception { Document declaredAnnotation = (Document) o.getClass().getDeclaredAnnotation(Document.class); if (declaredAnnotation == null) { throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", o.getClass().getName())); } String indexName = declaredAnnotation.indexName(); IndexRequest request = new IndexRequest(indexName); Field fieldByAnnotation = getFieldByAnnotation(o, EsDataId.class); if (fieldByAnnotation != null) { fieldByAnnotation.setAccessible(true); try { Object id = fieldByAnnotation.get(o); request = request.id(id.toString()); } catch (IllegalAccessException e) { log.error("获取id字段出错:{}", e); } } String userJson = JSON.toJSONString(o); request.source(userJson, XContentType.JSON); IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); return indexResponse; } /** * 根据id查询 * * @return */ public String queryById(String indexName, String id) throws IOException { GetRequest getRequest = new GetRequest(indexName, id); // getRequest.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); GetResponse getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT); String jsonStr = getResponse.getSourceAsString(); return jsonStr; } /** * 查询封装返回json字符串 * * @param indexName * @param searchSourceBuilder * @return * @throws IOException */ public String search(String indexName, SearchSourceBuilder searchSourceBuilder) throws IOException { SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(searchSourceBuilder); searchRequest.scroll(TimeValue.timeValueMinutes(1L)); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); SearchHits hits = searchResponse.getHits(); JSONArray jsonArray = new JSONArray(); for (SearchHit hit : hits) { String sourceAsString = hit.getSourceAsString(); JSONObject jsonObject = JSON.parseObject(sourceAsString); jsonArray.add(jsonObject); } log.info("返回总数为:" + hits.getTotalHits()); return jsonArray.toJSONString(); } /** * 查询封装,带分页 * * @param searchSourceBuilder * @param pageNum * @param pageSize * @param s * @param * @return * @throws IOException */ public <T> PageInfo<T> search(SearchSourceBuilder searchSourceBuilder, int pageNum, int pageSize, Class<T> s) throws Exception { Document declaredAnnotation = (Document) s.getDeclaredAnnotation(Document.class); if (declaredAnnotation == null) { throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName())); } String indexName = declaredAnnotation.indexName(); SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); SearchHits hits = searchResponse.getHits(); JSONArray jsonArray = new JSONArray(); for (SearchHit hit : hits) { String sourceAsString = hit.getSourceAsString(); JSONObject jsonObject = JSON.parseObject(sourceAsString); jsonArray.add(jsonObject); } log.info("返回总数为:" + hits.getTotalHits()); int total = (int) hits.getTotalHits().value; // 封装分页 List<T> list = jsonArray.toJavaList(s); PageInfo<T> page = new PageInfo<>(); page.setList(list); page.setPageNum(pageNum); page.setPageSize(pageSize); page.setTotal(total); page.setPages(total == 0 ? 0 : (total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1)); page.setHasNextPage(page.getPageNum() < page.getPages()); return page; } /** * 查询封装,返回集合 * * @param searchSourceBuilder * @param s * @param * @return * @throws IOException */ public <T> List<T> search(SearchSourceBuilder searchSourceBuilder, Class<T> s) throws Exception { Document declaredAnnotation = (Document) s.getDeclaredAnnotation(Document.class); if (declaredAnnotation == null) { throw new Exception(String.format("class name: %s can not find Annotation [Document], please check", s.getName())); } String indexName = declaredAnnotation.indexName(); SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(searchSourceBuilder); searchRequest.scroll(TimeValue.timeValueMinutes(1L)); SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); // //配置标题高亮显示 // HighlightBuilder highlightBuilder = new HighlightBuilder(); //生成高亮查询器 // highlightBuilder.field(title); //高亮查询字段 // highlightBuilder.field(content); //高亮查询字段 // highlightBuilder.requireFieldMatch(false); //如果要多个字段高亮,这项要为false // highlightBuilder.preTags(""); //高亮设置 // highlightBuilder.postTags(""); // // //下面这两项,如果你要高亮如文字内容等有很多字的字段,必须配置,不然会导致高亮不全,文章内容缺失等 // highlightBuilder.fragmentSize(800000); //最大高亮分片数 // highlightBuilder.numOfFragments(0); //从第一个分片获取高亮片段 String scrollId = searchResponse.getScrollId(); SearchHits hits = searchResponse.getHits(); JSONArray jsonArray = new JSONArray(); for (SearchHit hit : hits) { String sourceAsString = hit.getSourceAsString(); JSONObject jsonObject = JSON.parseObject(sourceAsString); jsonArray.add(jsonObject); } // 封装分页 List<T> list = jsonArray.toJavaList(s); return list; } /** * 批量插入文档 * 文档存在 则插入 * 文档不存在 则更新 * * @param list * @return */ public <T> boolean batchSaveOrUpdate(List<T> list, boolean izAsync) throws Exception { Object o1 = list.get(0); Document declaredAnnotation = (Document) o1.getClass().getDeclaredAnnotation(Document.class); if (declaredAnnotation == null) { throw new Exception(String.format("class name: %s can not find Annotation [@Document], please check", o1.getClass().getName())); } String indexName = declaredAnnotation.indexName(); BulkRequest request = new BulkRequest(indexName); for (Object o : list) { String jsonStr = JSON.toJSONString(o); IndexRequest indexReq = new IndexRequest().source(jsonStr, XContentType.JSON); Field fieldByAnnotation = getFieldByAnnotation(o, EsDataId.class); if (fieldByAnnotation != null) { fieldByAnnotation.setAccessible(true); try { Object id = fieldByAnnotation.get(o); indexReq = indexReq.id(id.toString()); } catch (IllegalAccessException e) { log.error("获取id字段出错:{}", e); } } request.add(indexReq); } if (izAsync) { BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); return outResult(bulkResponse); } else { restHighLevelClient.bulkAsync(request, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() { @Override public void onResponse(BulkResponse bulkResponse) { outResult(bulkResponse); } @Override public void onFailure(Exception e) { log.error("es数据 添加错误{}", e); } }); } return true; } private boolean outResult(BulkResponse bulkResponse) { for (BulkItemResponse bulkItemResponse : bulkResponse) { DocWriteResponse itemResponse = bulkItemResponse.getResponse(); IndexResponse indexResponse = (IndexResponse) itemResponse; log.info("单条返回结果:{}", indexResponse); if (bulkItemResponse.isFailed()) { log.error("es 返回错误{}", bulkItemResponse.getFailureMessage()); return false; } } return true; } /** * 删除文档 * * @param indexName: 索引名称 * @param docId: 文档id */ public boolean deleteDoc(String indexName, String docId) throws IOException { DeleteRequest request = new DeleteRequest(indexName, docId); DeleteResponse deleteResponse = restHighLevelClient.delete(request, RequestOptions.DEFAULT); // 解析response String index = deleteResponse.getIndex(); String id = deleteResponse.getId(); long version = deleteResponse.getVersion(); ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); if (shardInfo.getFailed() > 0) { for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { String reason = failure.reason(); log.info("删除失败,原因为 {}", reason); } } return true; } /** * 根据json类型更新文档 * * @param indexName * @param docId * @param o * @return * @throws IOException */ public boolean updateDoc(String indexName, String docId, Object o) throws IOException { UpdateRequest request = new UpdateRequest(indexName, docId); request.doc(JSON.toJSONString(o), XContentType.JSON); UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT); String index = updateResponse.getIndex(); String id = updateResponse.getId(); long version = updateResponse.getVersion(); if (updateResponse.getResult() == DocWriteResponse.Result.CREATED) { return true; } else if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) { return true; } else if (updateResponse.getResult() == DocWriteResponse.Result.DELETED) { } else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) { } return false; } /** * 根据Map类型更新文档 * * @param indexName * @param docId * @param map * @return * @throws IOException */ public boolean updateDoc(String indexName, String docId, Map<String, Object> map) throws IOException { UpdateRequest request = new UpdateRequest(indexName, docId); request.doc(map); UpdateResponse updateResponse = restHighLevelClient.update(request, RequestOptions.DEFAULT); String index = updateResponse.getIndex(); String id = updateResponse.getId(); long version = updateResponse.getVersion(); if (updateResponse.getResult() == DocWriteResponse.Result.CREATED) { return true; } else if (updateResponse.getResult() == DocWriteResponse.Result.UPDATED) { return true; } else if (updateResponse.getResult() == DocWriteResponse.Result.DELETED) { } else if (updateResponse.getResult() == DocWriteResponse.Result.NOOP) { } return false; } public XContentBuilder generateBuilder(Class clazz) throws IOException { // 获取索引名称及类型 Document doc = (Document) clazz.getAnnotation(Document.class); System.out.println(doc.indexName()); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject("properties"); Field[] declaredFields = clazz.getDeclaredFields(); for (Field f : declaredFields) { if (f.isAnnotationPresent(org.jeecg.common.aspect.annotation.es.Field.class)) { // 获取注解 org.jeecg.common.aspect.annotation.es.Field declaredAnnotation = f.getDeclaredAnnotation(org.jeecg.common.aspect.annotation.es.Field.class); // 如果嵌套对象: /** * { * "mappings": { * "properties": { * "region": { * "type": "keyword" * }, * "manager": { * "properties": { * "age": { "type": "integer" }, * "name": { * "properties": { * "first": { "type": "text" }, * "last": { "type": "text" } * } * } * } * } * } * } * } */ if (declaredAnnotation.type() == FieldType.OBJECT) { // 获取当前类的对象-- Action Class<?> type = f.getType(); Field[] df2 = type.getDeclaredFields(); builder.startObject(f.getName()); builder.startObject("properties"); // 遍历该对象中的所有属性 for (Field f2 : df2) { if (f2.isAnnotationPresent(org.jeecg.common.aspect.annotation.es.Field.class)) { // 获取注解 org.jeecg.common.aspect.annotation.es.Field declaredAnnotation2 = f2.getDeclaredAnnotation(org.jeecg.common.aspect.annotation.es.Field.class); builder.startObject(f2.getName()); builder.field("type", declaredAnnotation2.type().getType()); // keyword不需要分词 if (declaredAnnotation2.type() == FieldType.TEXT) { builder.field("analyzer", declaredAnnotation2.analyzer().getType()); } if (declaredAnnotation2.type() == FieldType.DATE) { builder.field("format", "yyyy-MM-dd HH:mm:ss"); } builder.endObject(); } } builder.endObject(); builder.endObject(); } else { builder.startObject(f.getName()); builder.field("type", declaredAnnotation.type().getType()); // keyword不需要分词 if (declaredAnnotation.type() == FieldType.TEXT) { builder.field("analyzer", declaredAnnotation.analyzer().getType()); } if (declaredAnnotation.type() == FieldType.DATE) { builder.field("format", "yyyy-MM-dd HH:mm:ss"); } builder.endObject(); } } } // 对应property builder.endObject(); builder.endObject(); return builder; } public static Field getFieldByAnnotation(Object o, Class annotationClass) { Field[] declaredFields = o.getClass().getDeclaredFields(); if (declaredFields != null && declaredFields.length > 0) { for (Field f : declaredFields) { if (f.isAnnotationPresent(annotationClass)) { return f; } } } return null; } /** * 获取低水平客户端 * * @return */ public RestClient getLowLevelClient() { return restHighLevelClient.getLowLevelClient(); } /** * 高亮结果集 特殊处理 * map转对象 JSONObject.parseObject(JSONObject.toJSONString(map), Content.class) * * @param searchResponse * @param highlightField */ public List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) { //解析结果 ArrayList<Map<String, Object>> list = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits().getHits()) { Map<String, HighlightField> high = hit.getHighlightFields(); HighlightField title = high.get(highlightField); hit.getSourceAsMap().put("id", hit.getId()); Map<String, Object> sourceAsMap = hit.getSourceAsMap();//原来的结果 //解析高亮字段,将原来的字段换为高亮字段 if (title != null) { Text[] texts = title.fragments(); String nTitle = ""; for (Text text : texts) { nTitle += text; } //替换 sourceAsMap.put(highlightField, nTitle); } list.add(sourceAsMap); } return list; } /** * 查询并分页 * * @param index 索引名称 * @param query 查询条件 * @param size 文档大小限制 * @param from 从第几页开始 * @param fields 需要显示的字段,逗号分隔(缺省为全部字段) * @param sortField 排序字段 * @param highlightField 高亮字段 * @return */ public List<Map<String, Object>> searchListData(String index, SearchSourceBuilder query, Integer size, Integer from, String fields, String sortField, String highlightField) throws IOException { SearchRequest request = new SearchRequest(index); SearchSourceBuilder builder = query; if (StringUtils.isNotEmpty(fields)) { //只查询特定字段。如果需要查询所有字段则不设置该项。 builder.fetchSource(new FetchSourceContext(true, fields.split(","), Strings.EMPTY_ARRAY)); } from = from <= 0 ? 0 : from * size; //设置确定结果要从哪个索引开始搜索的from选项,默认为0 builder.from(from); builder.size(size); if (StringUtils.isNotEmpty(sortField)) { //排序字段,注意如果proposal_no是text类型会默认带有keyword性质,需要拼接.keyword builder.sort(sortField + ".keyword", SortOrder.ASC); } //高亮 HighlightBuilder highlight = new HighlightBuilder(); highlight.field(highlightField); //关闭多个高亮 highlight.requireFieldMatch(false); highlight.preTags(""); highlight.postTags(""); builder.highlighter(highlight); //不返回源数据。只有条数之类的数据。 //builder.fetchSource(false); request.source(builder); SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT); log.error("==" + response.getHits().getTotalHits()); if (response.status().getStatus() == 200) { // 解析对象 return setSearchResponse(response, highlightField); } return null; } }

测试

pojo

import org.jeecg.common.aspect.annotation.es.Document;
import org.jeecg.common.aspect.annotation.es.EsDataId;
import org.jeecg.common.aspect.annotation.es.Field;
import org.jeecg.common.constant.enums.AnalyzerType;
import org.jeecg.common.constant.enums.FieldType;

import java.io.Serializable;
import java.util.Objects;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/20 14:17
 */
@Document(indexName = "word")
public class DocumentWord implements Serializable {

    private static final long serialVersionUID = 8272130486926494273L;

    @EsDataId
    @Field(type = FieldType.TEXT)
    private String id;

    @Field(type = FieldType.TEXT,analyzer = AnalyzerType.IK_MAX_WORD)
    private String title;

    @Field(type = FieldType.TEXT,analyzer = AnalyzerType.IK_SMART)
    private String content;

    public DocumentWord() {
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getContent() {
        return content;
    }

    public void setContent(String content) {
        this.content = content;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (!(o instanceof DocumentWord)) return false;
        DocumentWord that = (DocumentWord) o;
        return Objects.equals(getId(), that.getId()) && Objects.equals(getTitle(), that.getTitle()) && Objects.equals(getContent(), that.getContent());
    }

    @Override
    public int hashCode() {
        return Objects.hash(getId(), getTitle(), getContent());
    }

    @Override
    public String toString() {
        return "DocumentWord{" +
                "id='" + id + '\'' +
                ", title='" + title + '\'' +
                ", content='" + content + '\'' +
                '}';
    }
}

service


import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.formula.functions.T;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jeecg.common.aspect.annotation.es.Document;
import org.jeecg.common.util.ESUtil;
import org.jeecg.modules.demo.es.pojo.DocumentWord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.List;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/20 14:42
 */
@Service
@Slf4j
public class EsImpl<T> {

    @Autowired
    private ESUtil esUtil;

    public boolean removeIndex(String indexName){
        return esUtil.delIndex(indexName);
    }

    public boolean addData(List<T> list) {
        if(list.size()<0){
            return true;
        }
        try {
            if(esUtil.createIndex(list.get(0).getClass()))
                return esUtil.batchSaveOrUpdate(list,true);
            else
                return false;
        } catch (Exception e) {
            log.error("############### 数据添加失败! {}", e);
        }
        return false;
    }

    public List<DocumentWord> search(SearchSourceBuilder searchSourceBuilder) {
        try {
            return esUtil.search(searchSourceBuilder, DocumentWord.class);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

}

controller

package org.jeecg.modules.demo.es.controller;

import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.demo.es.pojo.DocumentWord;
import org.jeecg.modules.demo.es.service.EsImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;

import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * @author qinlei
 * @description todo
 * @date 2021/3/5 16:31
 */
@Api("es测试")
@RestController
@RequestMapping(("/es"))
public class SearchController {
    @Autowired
    private EsImpl<DocumentWord> searchServiceImp;

    @ApiOperation("删除索引")
    @PostMapping("/removeIndex/{indexName}")
    public Result removeIndex(@PathVariable("indexName") String indexName){
        return Result.OK(searchServiceImp.removeIndex(indexName));
    }

    @ApiOperation("新增数据")
    @PostMapping("addData")
    public Result addData(HttpServletRequest request){
        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
        MultipartFile multipartFile = multipartRequest.getFile("file");
        InputStreamReader isr = null;
        BufferedReader bReader = null;
        StringBuilder sb = new StringBuilder();
        try {
            isr = new InputStreamReader(multipartFile.getInputStream());
            bReader = new BufferedReader(isr);
            String s = "";
            while ((s =bReader.readLine()) != null) {
                sb.append(s + "\n");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if(bReader != null){
                    bReader.close();
                }
                if(isr != null){
                    isr.close();
                }
            } catch (IOException ioException) {
                ioException.printStackTrace();
            }
        }
        DocumentWord documentWord = new DocumentWord();
        documentWord.setId("1");
        documentWord.setTitle("******手册");
        documentWord.setContent(sb.toString());
        List<DocumentWord> list = new ArrayList<>();
        list.add(documentWord);
        return Result.OK(searchServiceImp.addData(list));
    }


    @ApiOperation("查询数据")
    @GetMapping("queryDataById")
    public Result queryDataById(@RequestParam("searchType") String searchType,
                                @RequestParam("searchValue") String searchValue){
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.from(0);
        searchSourceBuilder.size(5);
        // 符合条件查询
        BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
        boolBuilder.must(QueryBuilders.termQuery(searchType, searchValue));
        searchSourceBuilder.query(boolBuilder);
        return Result.OK(searchServiceImp.search(searchSourceBuilder));
    }
}

副本小坑理解

因为我这里只有一个节点,所以设置副本为0,当设置>0的时候,es健康状态为黄色

比如设置副本为1,分片为4,es实际分配时,当前节点分配2个分片,还有两个会寻找其他节点,由于节点只有一个,找不到其他的node,所以会报黄色健康

ik分词器的两个属性

ik_max_word:
会将文本做最细粒度的拆分;尽可能多的拆分出词语
用他其实就是可以进行最大程度的模糊查询
ik_smart:
会做最粗粒度的拆分;已被分出的词语将不会再次被其它词语占有
用它其实就是进行精确查找

参考

工具类及注解部分copy(做了小部分优化):

https://blog.csdn.net/lsqingfeng/article/details/107620558

其实核心有点类似模仿springdataelasticsearch封装

你可能感兴趣的:(java,中间件,elasticsearch,springboot,全文检索)