Spring Boot2.X集成ElasticSearch6.X 集群
转载请标明出处 https://blog.csdn.net/Amor_Leo/article/details/83012038 谢谢
ES集群搭建 https://blog.csdn.net/Amor_Leo/article/details/83011372 两个节点
ES集群搭建 https://blog.csdn.net/Amor_Leo/article/details/83144739 三个节点
该文参考了 https://blog.csdn.net/chy2z/article/details/80461745
Spring Boot2.X集成ElasticSearch6.X 集群
- pom.xml
- ES配置类
- 实体类
- 工具类
- Controller 测试类
- Spring Boot 启动报错
pom.xml
>
>org.springframework.boot >
>spring-boot-starter-parent >
>2.1.1.RELEASE >
> <!-- lookup parent from repository -->
>
>UTF-8 >
>UTF-8 >
>1.8 >
>
>
>
>org.springframework.boot >
>spring-boot-starter-web >
>
>
>org.elasticsearch.client >
>transport >
>6.5.4 >
>
>
>org.elasticsearch >
>elasticsearch >
>
>
>
>
>org.elasticsearch >
>elasticsearch >
>6.5.4 >
>
>
>org.elasticsearch.plugin >
>transport-netty4-client >
>6.5.4 >
>
<!-- swagger2 文档生成 -->
io.springfox
springfox-swagger2
2.9.2
io.springfox
springfox-swagger-ui
2.9.2
org.springframework.boot
spring-boot-configuration-processor
true
com.alibaba
fastjson
1.2.54
org.apache.commons
commons-lang3
3.8.1
org.springframework.boot
spring-boot-starter-test
test
>
>
>
>
>org.springframework.boot >
>spring-boot-maven-plugin >
>
>
>
ES配置类
/**
* @Auther: LHL
* @Date: 2018/10/10 08:55
* @Description: ES 连接 配置
*/
@Configuration
public class ESConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(ESConfig.class);
@Bean(name = "transportClient")
public TransportClient transportClient() {
LOGGER.info("Elasticsearch初始化开始。。。。。");
TransportClient transportClient = null;
try {
//es集群连接
TransportAddress node = new TransportAddress(
InetAddress.getByName("192.168.0.111"),
9300
);
TransportAddress node1 = new TransportAddress(
InetAddress.getByName("192.168.0.112"),
9300
);
TransportAddress node2 = new TransportAddress(
InetAddress.getByName("192.168.0.113"),
9300
);
//es集群配置（自定义配置） 连接自己安装的集群名称
Settings settings = Settings.builder()
.put("cluster.name", "ESCluster")
//增加嗅探机制，找到ES集群 如果报org.elasticsearch.transport.ReceiveTimeoutTransportException: 把他注释掉 即可
.put("client.transport.sniff", true)
//增加线程池个数，暂时设为5
.put("thread_pool.search.size", Integer.parseInt("5"))
.build();
transportClient = new PreBuiltTransportClient(settings);
transportClient.addTransportAddress(node);
transportClient.addTransportAddress(node1);
transportClient.addTransportAddress(node2);
} catch (UnknownHostException e) {
e.printStackTrace();
LOGGER.error("elasticsearch TransportClient create error!!", e);
}
return transportClient;
}
}
/**
* @Auther: LHL
* @Date: 2018/10/16 11:44
* @Description: ES批量操作初始化
*/
@Configuration
public class ESClient {
private static final Logger LOGGER = LogManager.getLogger(ESClient.class);
@Autowired
private TransportClient transportClient;
private static TransportClient client;
/**
* @PostContruct 是spring容器初始化的时候执行该方法
* @param: []
* @return: void
* @auther: LHL
* @date: 2018/10/16 14:19
*/
@PostConstruct
public void init() {
client = this.transportClient;
}
@Bean("bulkProcessor")
public BulkProcessor bulkProcessor() {
LOGGER.info("bulkProcessor初始化开始。。。。。");
return BulkProcessor.builder(client, new BulkProcessor.Listener() {
@Override
public void beforeBulk(long l, BulkRequest bulkRequest) {
}
@Override
public void afterBulk(long l, BulkRequest bulkRequest, BulkResponse bulkResponse) {
}
@Override
public void afterBulk(long l, BulkRequest bulkRequest, Throwable throwable) {
LOGGER.error("{} data bulk failed,reason :{}", bulkRequest.numberOfActions(), throwable);
}
}).setBulkActions(10000) // 批量导入个数
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) // 满5MB进行导入
.setFlushInterval(TimeValue.timeValueSeconds(5)) // 冲刷间隔
.setConcurrentRequests(3) // 并发数
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueSeconds(1), 3)) // 重试3次，间隔1s
.build();
}
}
实体类
/**
* @Auther: LHL
* @Date: 2018/10/10 11:03
* @Description:
*/
public class EsModel {
private String id;
private Integer age;
private String name;
private Date date;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Integer getAge() {
return age;
}
public void setAge(Integer age) {
this.age = age;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
@Override
public String toString() {
return "EsModel{" +
"id='" + id + '\'' +
", age=" + age +
", name='" + name + '\'' +
", date=" + date +
'}';
}
}
public class Book {
String id;
String name;
String message;
Double price;
Date creatDate;
public Book(String id, String name, String message, Double price, Date creatDate) {
this.id = id;
this.name = name;
this.message = message;
this.price = price;
this.creatDate = creatDate;
}
public Date getCreatDate() {
return creatDate;
}
public void setCreatDate(Date creatDate) {
this.creatDate = creatDate;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public Double getPrice() {
return price;
}
public void setPrice(Double price) {
this.price = price;
}
}
工具类
/**
* @Auther: LHL
* @Date: 2018/10/10 10:38
* @Description:
*/
public class EsPage {
/**
* 当前页
*/
private int currentPage;
/**
* 每页显示多少条
*/
private int pageSize;
/**
* 总记录数
*/
private int recordCount;
/**
* 本页的数据列表
*/
private List<Map<String, Object>> recordList;
/**
* 总页数
*/
private int pageCount;
/**
* 页码列表的开始索引（包含）
*/
private int beginPageIndex;
/**
* 页码列表的结束索引（包含）
*/
private int endPageIndex;
/**
* 只接受前4个必要的属性，会自动的计算出其他3个属性的值
* @param currentPage
* @param pageSize
* @param recordCount
* @param recordList
*/
public EsPage(int currentPage, int pageSize, int recordCount, List<Map<String, Object>> recordList) {
this.currentPage = currentPage;
this.pageSize = pageSize;
this.recordCount = recordCount;
this.recordList = recordList;
// 计算总页码
pageCount = (recordCount + pageSize - 1) / pageSize;
// 计算 beginPageIndex 和 endPageIndex // >> 总页数不多于10页，则全部显示
if (pageCount <= 10) {
beginPageIndex = 1;
endPageIndex = pageCount;
}
// >> 总页数多于10页，则显示当前页附近的共10个页码
else {
// 当前页附近的共10个页码（前4个 + 当前页 + 后5个）
beginPageIndex = currentPage - 4;
endPageIndex = currentPage + 5;
// 当前面的页码不足4个时，则显示前10个页码
if (beginPageIndex < 1) {
beginPageIndex = 1;
endPageIndex = 10;
}
// 当后面的页码不足5个时，则显示后10个页码
if (endPageIndex > pageCount) {
endPageIndex = pageCount;
beginPageIndex = pageCount - 10 + 1;
}
}
}
public int getCurrentPage() {
return currentPage;
}
public void setCurrentPage(int currentPage) {
this.currentPage = currentPage;
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getRecordCount() {
return recordCount;
}
public void setRecordCount(int recordCount) {
this.recordCount = recordCount;
}
public List<Map<String, Object>> getRecordList() {
return recordList;
}
public void setRecordList(List<Map<String, Object>> recordList) {
this.recordList = recordList;
}
public int getPageCount() {
return pageCount;
}
public void setPageCount(int pageCount) {
this.pageCount = pageCount;
}
public int getBeginPageIndex() {
return beginPageIndex;
}
public void setBeginPageIndex(int beginPageIndex) {
this.beginPageIndex = beginPageIndex;
}
public int getEndPageIndex() {
return endPageIndex;
}
public void setEndPageIndex(int endPageIndex) {
this.endPageIndex = endPageIndex;
}
}
/**
* @Auther: LHL
* @Date: 2018/10/10 10:20
* @Description:
*/
@Component
public class ElasticsearchUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchUtil.class);
@Autowired
private TransportClient transportClient;
@Autowired
private BulkProcessor bulkProcessor;
private static TransportClient client;
private static BulkProcessor bulk;
/**
* @PostContruct是spring框架的注解 spring容器初始化的时候执行该方法
*/
@PostConstruct
public void init() {
client = this.transportClient;
}
/**
* @PostContruct是spring框架的注解 spring容器初始化的时候执行该方法
*/
@PostConstruct
public void initBulk() {
bulk = this.bulkProcessor;
}
/**
* 创建索引以及映射mapping，并给索引某些字段指定iK分词，以后向该索引中查询时，就会用ik分词。
* @param: [indexName, esTpye]
* @return: boolean
* @auther: LHL
* @date: 2018/10/15 17:04
*/
public static boolean createIndex(String indexName, String esTpye) {
if (!isIndexExist(indexName)) {
LOGGER.info("Index is not exits!");
}
//创建映射
XContentBuilder mapping = null;
try {
mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties") // .startObject("m_id").field("type","keyword").endObject() //title:字段名， type:文本类型 analyzer ：分词器类型
.startObject("id").field("type", "text").field("analyzer", "standard").endObject() //该字段添加的内容，查询时将会使用ik_smart分词
.startObject("name").field("type", "text").field("analyzer", "standard").endObject() //ik_smart ik_max_word standard
.startObject("message").field("type", "text").field("analyzer", "standard").endObject()
.startObject("price").field("type", "float").endObject()
.startObject("creatDate").field("type", "date").endObject()
.endObject()
.endObject();
} catch (IOException e) {
e.printStackTrace();
}
//index：索引名 type：类型名
PutMappingRequest putmap = Requests.putMappingRequest(indexName).type(esTpye).source(mapping);
//创建索引
client.admin().indices().prepareCreate(indexName).execute().actionGet();
//为索引添加映射
AcknowledgedResponse acknowledgedResponse = client.admin().indices().putMapping(putmap).actionGet();
LOGGER.info("执行建立成功？" + acknowledgedResponse.isAcknowledged());
return acknowledgedResponse.isAcknowledged();
}
/**
* 创建索引
* @param index
* @return
*/
public static boolean createIndex(String index) {
if (!isIndexExist(index)) {
LOGGER.info("Index is not exits!");
}
CreateIndexResponse indexresponse = client.admin().indices().prepareCreate(index).execute().actionGet();
LOGGER.info("执行建立成功？" + indexresponse.isAcknowledged());
return indexresponse.isAcknowledged();
}
/**
* 删除索引
* @param index
* @return
*/
public static boolean deleteIndex(String index) {
if (!isIndexExist(index)) {
LOGGER.info("Index is not exits!");
}
AcknowledgedResponse dResponse = client.admin().indices().prepareDelete(index).execute().actionGet();
if (dResponse.isAcknowledged()) {
LOGGER.info("delete index " + index + " successfully!");
} else {
LOGGER.info("Fail to delete index " + index);
}
return dResponse.isAcknowledged();
}
/**
* 判断索引是否存在
* @param index
* @return
*/
public static boolean isIndexExist(String index) {
IndicesExistsResponse inExistsResponse = client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet();
if (inExistsResponse.isExists()) {
LOGGER.info("Index [" + index + "] is exist!");
} else {
LOGGER.info("Index [" + index + "] is not exist!");
}
return inExistsResponse.isExists();
}
/**
* 数据添加，正定ID
* @param jsonObject 要增加的数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID(为空默认生成)
* @return
*/
public static String addData(JSONObject jsonObject, String index, String type, String id) {
IndexResponse response = client.prepareIndex(index, type, id).setSource(jsonObject).get();
LOGGER.info("addData response status:{},id:{}", response.status().getStatus(), response.getId());
return response.getId();
}
/**
* 数据添加
* @param jsonObject 要增加的数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @return
*/
public static String addData(JSONObject jsonObject, String index, String type) {
return addData(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase());
}
/**
* 通过ID删除数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID
*/
public static void deleteDataById(String index, String type, String id) {
DeleteResponse response = client.prepareDelete(index, type, id).execute().actionGet();
LOGGER.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId());
}
/**
* 通过ID 更新数据
* @param jsonObject 要增加的数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID
* @return
*/
public static void updateDataById(JSONObject jsonObject, String index, String type, String id) {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index).type(type).id(id).doc(jsonObject);
client.update(updateRequest);
}
/**
* 通过ID获取数据
* @param index 索引，类似数据库
* @param type 类型，类似表
* @param id 数据ID
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @return 结果
*/
public static Map<String, Object> searchDataById(String index, String type, String id, String fields) {
GetRequestBuilder getRequestBuilder = client.prepareGet(index, type, id);
if (StringUtils.isNotEmpty(fields)) {
getRequestBuilder.setFetchSource(fields.split(","), null);
}
GetResponse getResponse = getRequestBuilder.execute().actionGet();
return getResponse.getSource();
}
/**
* 批量增加
* @param: [index, type, bookList]
* @return: org.elasticsearch.action.bulk.BulkResponse
* @auther: LHL
* @date: 2018/10/15 13:12
*/
public static void bulkAddDocument(String index, String type, List<Book> bookList) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
bookList.stream().forEach(
book -> {
try {
bulkRequest.add(client.prepareIndex(index, type, book.getId())
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("id", book.getId())
.field("name", book.getName())
.field("creatDate", book.getCreatDate())
.field("price", book.getPrice())
.field("message", book.getMessage())
.endObject()
)
);
if (Integer.valueOf(book.getId()) % 100000 == 0) {
BulkResponse responses = bulkRequest.execute().actionGet();
if (responses.hasFailures()) {
System.out.println("bulk error:" + responses.buildFailureMessage());
}
LOGGER.info("insert status?" + responses.status());
}
} catch (IOException e) {
e.printStackTrace();
}
}
);
BulkResponse responses = bulkRequest.execute().actionGet();
if (responses.hasFailures()) {
System.out.println("bulk error:" + responses.buildFailureMessage());
}
LOGGER.info("insert status?" + responses.status());
}
/**
* bulkProcessor 批量增加
* @param: [indexName, type, bookList]
* @return: void
* @auther: LHL
* @date: 2018/10/16 11:46
*/
public static void bulkProcessorAdd(String indexName, String type, List<Book> bookList) {
bookList.stream().parallel().forEach(
book -> {
try {
bulk.add(new IndexRequest(indexName, type, book.getId()).source(XContentFactory.jsonBuilder()
.startObject()
.field("id", book.getId())
.field("name", book.getName())
.field("creatDate", book.getCreatDate())
.field("price", book.getPrice())
.field("message", book.getMessage())
.endObject()));
} catch (IOException e) {
e.printStackTrace();
}
}
);
}
/**
* 批量删除
* @param: [index, type, ids]
* @return: org.elasticsearch.action.bulk.BulkResponse
* @auther: LHL
* @date: 2018/10/15 11:42
*/
public static BulkResponse bulkDeleteDocument(String index, String type, List<String> ids) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
ids.stream().forEach(
id -> bulkRequest.add(client.prepareDelete(index, type, id))
);
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
return bulkResponse;
}
/**
* 查询所有
* @param: [indexName]
* @return: void
* @auther: LHL
* @date: 2018/10/15 14:56
*/
public static List<String> searchAll(String indexName) {
List<String> resultList = new ArrayList<>();
String result = null;
SearchResponse response = client.prepareSearch(indexName).setQuery(matchAllQuery()).setFrom(0).setSize(10).addSort("price", SortOrder.ASC).get();
for (SearchHit searchHit : response.getHits()) {
result = searchHit.getSourceAsString();
resultList.add(result);
}
return resultList;
}
/**
* 使用分词查询 高亮 排序 ,并分页
* @param index 索引名称
* @param type 类型名称,可传入多个type逗号分隔
* @param startPage 当前页
* @param pageSize 每页显示条数
* @param query 查询条件
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @param sortField 排序字段
* @param highlightField 高亮字段
* @return 结果
*/
public static EsPage searchDataPage(String index, String type, int startPage, int pageSize, QueryBuilder query, String fields, String sortField, String highlightField) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
if (StringUtils.isNotEmpty(type)) {
searchRequestBuilder.setTypes(type.split(","));
}
// 需要显示的字段，逗号分隔（缺省为全部字段）
if (StringUtils.isNotEmpty(fields)) {
searchRequestBuilder.setFetchSource(fields.split(","), null);
}//排序字段
if (StringUtils.isNotEmpty(sortField)) {
searchRequestBuilder.addSort(sortField, SortOrder.ASC);
}// 高亮（xxx=111,aaa=222）
if (StringUtils.isNotEmpty(highlightField)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.preTags("");//设置前缀
highlightBuilder.postTags("");//设置后缀
// 设置高亮字段
highlightBuilder.field(highlightField);
searchRequestBuilder.highlighter(highlightBuilder);
}
// 设置是否按查询匹配度排序
searchRequestBuilder.setExplain(true);
//如果 pageSize是10 那么startPage>9990 (10000-pagesize) 如果 20 那么 >9980 如果 50 那么>9950
//深度分页 TODO
if (startPage > (10000-pageSize)) {
searchRequestBuilder.setQuery(query);
searchRequestBuilder
.setScroll(TimeValue.timeValueMinutes(1))
.setSize(10000);
//打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询
LOGGER.info("\n{}", searchRequestBuilder);
// 执行搜索,返回搜索响应信息
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
if (searchResponse.status().getStatus() == 200){
//使用scrollId迭代查询
List<Map<String, Object>> result = disposeScrollResult(searchResponse, highlightField);
List<Map<String, Object>> sourceList = result.stream().parallel().skip((startPage - 1- (10000/pageSize)) * pageSize).limit(pageSize).collect(Collectors.toList());
return new EsPage(startPage, pageSize, (int) totalHits, sourceList);
}
} else {//浅度分页
searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // QUERY_THEN_FETCH QUERY_AND_FETCH DFS_QUERY_THEN_FETCH
searchRequestBuilder.setQuery(matchAllQuery());
searchRequestBuilder.setQuery(query);
// 分页应用
searchRequestBuilder
.setFrom(startPage)
.setSize(pageSize);
//打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询
LOGGER.info("\n{}", searchRequestBuilder);
// 执行搜索,返回搜索响应信息
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
long length = searchResponse.getHits().getHits().length;
LOGGER.debug("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length);
if (searchResponse.status().getStatus() == 200) {
// 解析对象
List<Map<String, Object>> sourceList = setSearchResponse(searchResponse, highlightField);
return new EsPage(startPage, pageSize, (int) totalHits, sourceList);
}
}
return null;
}
/**
* 深度分页 从当前页为1001开始
* @param: [indexName, esType, startPage, pageSize, highlightField]
* @return: com.aqh.utils.EsPage
* @auther: LHL
* @date: 2018/10/17 15:07
*/
public static EsPage deepPageing(String indexName, String esType, int startPage, int pageSize, String highlightField) {
System.out.println("scroll 模式启动！");
long begin = System.currentTimeMillis();
//初始化查询，获取scrollId
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name", "名"));
// boolQueryBuilder.filter(QueryBuilders.rangeQuery("price").from("1").to("999821"));
SearchResponse response = client.prepareSearch(indexName)//对应索引
.setTypes(esType)//对应索引type
.setQuery(boolQueryBuilder)
.addSort("price", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(1))
.setSize(10000) //第一次不返回size条数据
.highlighter(new HighlightBuilder().preTags("").postTags("").field(highlightField))
.setExplain(true)
.execute()
.actionGet();
long totalHits = response.getHits().totalHits;
List<Map<String, Object>> result = disposeScrollResult(response, highlightField);
List<Map<String, Object>> sourceList = result.stream().parallel().skip((startPage - 1-(10000/pageSize)) * pageSize).limit(pageSize).collect(Collectors.toList());
long end = System.currentTimeMillis();
System.out.println("耗时: " + (end - begin) + "ms");
System.out.println("耗时: " + (end - begin) / 1000 + "s");
System.out.println("查询"+totalHits+"条数据");
return new EsPage(startPage, pageSize, (int) totalHits, sourceList);
}
/**
* 处理scroll结果
* @param: [response, highlightField]
* @return: java.util.List>
* @auther: LHL
* @date: 2018/10/17 11:17
*/
private static List<Map<String, Object>> disposeScrollResult(SearchResponse response ,String highlightField){
List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
//使用scrollId迭代查询
while (response.getHits().getHits().length > 0) {
String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId)
.setScroll(TimeValue.timeValueMinutes(1))//设置查询context的存活时间
.execute()
.actionGet();
SearchHits hits = response.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> resultMap =getResultMap(hit, highlightField);
sourceList.add(resultMap);
// System.out.println(JSON.toJSONString(resultMap));
}
}
ClearScrollRequest request = new ClearScrollRequest();
request.addScrollId(response.getScrollId());
client.clearScroll(request);
return sourceList;
}
/**
* 使用分词查询 排序 高亮
* @param index 索引名称
* @param type 类型名称,可传入多个type逗号分隔
* @param query 查询条件
* @param size 文档大小限制
* @param fields 需要显示的字段，逗号分隔（缺省为全部字段）
* @param sortField 排序字段
* @param highlightField 高亮字段
* @return 结果
*/
public static List<Map<String, Object>> searchListData(String index, String type, QueryBuilder query, Integer size, String fields, String sortField, String highlightField) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
if (StringUtils.isNotEmpty(type)) {
searchRequestBuilder.setTypes(type.split(","));
}
if (StringUtils.isNotEmpty(highlightField)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
// 设置高亮字段
highlightBuilder.field(highlightField);
searchRequestBuilder.highlighter(highlightBuilder);
}
searchRequestBuilder.setQuery(query);
if (StringUtils.isNotEmpty(fields)) {
searchRequestBuilder.setFetchSource(fields.split(","), null);
}
searchRequestBuilder.setFetchSource(true);
if (StringUtils.isNotEmpty(sortField)) {
searchRequestBuilder.addSort(sortField, SortOrder.ASC);
}
if (size != null && size > 0) {
searchRequestBuilder.setSize(size);
}//打印的内容 可以在 Elasticsearch head 和 Kibana 上执行查询
LOGGER.info("\n{}", searchRequestBuilder);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
long length = searchResponse.getHits().getHits().length;
LOGGER.info("共查询到[{}]条数据,处理数据条数[{}]", totalHits, length);
if (searchResponse.status().getStatus() == 200) {
// 解析对象
return setSearchResponse(searchResponse, highlightField);
}
return null;
}
/**
* 高亮结果集 特殊处理
* @param searchResponse 搜索的结果集
* @param highlightField 高亮字段
*/
private static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) {
List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
Map<String, Object> resultMap = getResultMap(searchHit, highlightField);
sourceList.add(resultMap);
}
return sourceList;
}
/**
* 获取高亮结果集
* @param: [hit, highlightField]
* @return: java.util.Map
* @auther: LHL
* @date: 2018/10/17 10:54
*/
private static Map<String, Object> getResultMap(SearchHit hit,String highlightField){
hit.getSourceAsMap().put("id", hit.getId());
if (StringUtils.isNotEmpty(highlightField)) {
Text[] text = hit.getHighlightFields().get(highlightField).getFragments();
String hightStr = null;
if (text != null) {
for (Text str : text) {
hightStr = str.string();
}
//遍历 高亮结果集，覆盖 正常结果集
hit.getSourceAsMap().put(highlightField, hightStr);
}
}
return hit.getSourceAsMap();
}
/**
* ik分词测试
* @param: []
* @return: void
* @auther: LHL
* @date: 2018/10/15 11:31
*/
public static String ik() {
StringBuilder stringBuilder = new StringBuilder();
AnalyzeRequest analyzeRequest = new AnalyzeRequest("entity")
.text("我是程序媛")
.analyzer("ik_max_word"); //ik_smart ik_max_word standard
List<AnalyzeResponse.AnalyzeToken> tokens = client.admin().indices()
.analyze(analyzeRequest)
.actionGet()
.getTokens();
for (AnalyzeResponse.AnalyzeToken token : tokens) {
stringBuilder.append(token.getTerm() + "\\r\\n");
}
return stringBuilder.toString();
}
}
Controller 测试类
/**
* @Auther: LHL
* @Date: 2018/10/10 10:12
* @Description:
*/
@RestController
@RequestMapping("/test")
public class TestController {
@GetMapping("/hello")
public String helloTest() {
return "Holle World";
}
/**
* 测试索引
*/
private String indexName = "entity";
/**
* 类型
*/
private String esType = "book";
/**
* http://127.0.0.1:8080/test/createIndex
* 创建索引
* @param request
* @param response
* @return
*/
@PutMapping("/createIndex")
public String createIndex(HttpServletRequest request, HttpServletResponse response) {
if (!ElasticsearchUtil.isIndexExist(indexName)) {
ElasticsearchUtil.createIndex(indexName);
} else {
return "索引已经存在";
}
return "索引创建成功";
}
/**
* 创建索引以及类型，并给索引某些字段指定iK分词，以后向该索引中查询时，就会用ik分词。
* @param: [request, response]
* @return: java.lang.String
* @auther: LHL
* @date: 2018/10/15 17:11
*/
@PutMapping("/createIndexTypeMapping")
public String createIndexTypeMapping(HttpServletRequest request, HttpServletResponse response) {
if (!ElasticsearchUtil.isIndexExist(indexName)) {
ElasticsearchUtil.createIndex(indexName,esType);
} else {
return "索引已经存在";
}
return "索引创建成功";
}
@DeleteMapping("/deleteIndex")
public String deleteIndex(String indexName, HttpServletRequest request, HttpServletResponse response) {
boolean b = ElasticsearchUtil.deleteIndex(indexName);
if (!b){
return "失败";
}
return "索引删除成功";
}
/**
* ik分词测试
* @param: []
* @return: void
* @auther: LHL
* @date: 2018/10/11 15:13
*/
@GetMapping("getik")
public String ikMapping(){
String ik = ElasticsearchUtil.ik();
return ik;
}
/**
* 插入记录
* @return
*/
@PostMapping("/insertJson")
public String insertJson() {
JSONObject jsonOject = new JSONObject();
jsonOject.put("id", DateFormatUtils.format(new Date(),"yyyyMMddhhmmss"));
jsonOject.put("age", 25);
jsonOject.put("name", "j-" + new Random(100).nextInt());
jsonOject.put("date", new Date());
String id = ElasticsearchUtil.addData(jsonOject, indexName, esType, jsonOject.getString("id"));
return id;
}
/**
* 插入记录
* @return
*/
@PostMapping("/insertModel")
public String insertModel() {
EsModel esModel = new EsModel();
esModel.setId(DateFormatUtils.format(new Date(),"yyyyMMddhhmmss"));
esModel.setName("m-" + new Random(100).nextInt());
esModel.setAge(30);
esModel.setDate(new Date());
JSONObject jsonObject = (JSONObject) JSONObject.toJSON(esModel);
String id = ElasticsearchUtil.addData(jsonObject, indexName, esType, jsonObject.getString("id"));
return id;
}
/**
* 删除记录
* @return
*/
@DeleteMapping("/delete")
public String delete(String id) {
if (StringUtils.isNotBlank(id)) {
ElasticsearchUtil.deleteDataById(indexName, esType, id);
return "删除id=" + id;
} else {
return "id为空";
}
}
/**
* 更新数据
* @return
*/
@PutMapping("/update/{id}")
public String update(@PathVariable("id") String id) {
if (StringUtils.isNotBlank(id)) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", id);
jsonObject.put("age", 31);
jsonObject.put("name", "修改");
jsonObject.put("date", new Date());
ElasticsearchUtil.updateDataById(jsonObject, indexName, esType, id);
return "id=" + id;
} else {
return "id为空";
}
}
/**
* 获取数据
* http://127.0.0.1:8080/test/getData/id
* @param id
* @return
*/
@GetMapping("/getData/{id}")
public String getData(@PathVariable("id") String id) {
if (StringUtils.isNotBlank(id)) {
Map<String, Object> map = ElasticsearchUtil.searchDataById(indexName, esType, id, null);
return JSONObject.toJSONString(map);
} else {
return "id为空";
}
}
/**
* 查询数据
* 模糊查询
* @return
*/
@GetMapping("/queryMatchData")
public String queryMatchData() {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolean matchPhrase = false;
if (matchPhrase == Boolean.TRUE) {
//不进行分词搜索
boolQuery.must(QueryBuilders.matchPhraseQuery("name", "修改书名"));
} else {
boolQuery.must(QueryBuilders.matchQuery("name", "修改书名"));
}
List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);
return JSONObject.toJSONString(list);
}
/**
* 通配符查询数据
* 通配符查询 ?用来匹配1个任意字符，*用来匹配零个或者多个字符
* @return
*/
@GetMapping("/queryWildcardData")
public String queryWildcardData() {
QueryBuilder queryBuilder = QueryBuilders.wildcardQuery("name.keyword", "书名*2");
List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, queryBuilder, 10, null, null, null);
return JSONObject.toJSONString(list);
}
/**
* 正则查询
* @return
*/
@GetMapping("/queryRegexpData")
public String queryRegexpData() {
QueryBuilder queryBuilder = QueryBuilders.regexpQuery("name.keyword", "书名[0-9]{1,7}");
List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, queryBuilder, 10, null, null, null);
return JSONObject.toJSONString(list);
}
/**
* 查询数字范围数据
* @return
*/
@GetMapping("/queryIntRangeData")
public String queryIntRangeData() {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.rangeQuery("price").from(21.0).to(25.0)
.includeLower(true) // true 包含下界， false 不包含下界
.includeUpper(false)); // true 包含下界， false 不包含下界
List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);
return JSONObject.toJSONString(list);
}
/**
* 查询日期范围数据
* @return
*/
@GetMapping("/queryDateRangeData")
public String queryDateRangeData() {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.rangeQuery("creatDate").from("2018-10-17T02:03:08.829Z").to("2018-10-17T02:03:09.727Z").includeLower(true).includeUpper(true));
List<Map<String, Object>> list = ElasticsearchUtil.searchListData(indexName, esType, boolQuery, 10, null, null, null);
return JSONObject.toJSONString(list);
}
/**
* 查询分页 高亮 排序
* @param startPage 第几条记录开始
* 从0开始
* @param pageSize 每页大小
* @return
*/
@GetMapping("/queryPage")
public String queryPage(String startPage, String pageSize ,String context) {
if (StringUtils.isNotBlank(startPage) && StringUtils.isNotBlank(pageSize)) {
long start = System.currentTimeMillis();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.rangeQuery("creatDate").from("2018-10-17T02:03:08.829Z").to("2018-10-17T02:03:09.727Z").includeLower(true).includeUpper(true))
.filter(QueryBuilders.matchQuery("name",context));
// boolQuery.filter(QueryBuilders.matchQuery("name",context));
EsPage list = ElasticsearchUtil.searchDataPage(indexName, esType, Integer.parseInt(startPage), Integer.parseInt(pageSize), boolQuery, null, "price", "name");
long end = System.currentTimeMillis();
System.out.println((end-start)/1000+"s");
System.out.println((end-start)+"ms");
return JSONObject.toJSONString(list);
} else {
return "startPage或者pageSize缺失";
}
}
/**
* 深度排序 分页 从当前页为1001开始
* @param: [startPage, pageSize]
* @return: com.aqh.utils.EsPage
* @auther: LHL
* @date: 2018/10/17 13:45
*/
@GetMapping("/deepPageing")
public EsPage deepPageing(int startPage, int pageSize){
if (startPage<(10000/pageSize)){
System.out.println("startPage需要>="+(10000/pageSize));
}
long start = System.currentTimeMillis();
EsPage result = ElasticsearchUtil.deepPageing(indexName, esType, startPage, pageSize, "name");
long end = System.currentTimeMillis();
System.out.println((end-start)/1000+"s");
System.out.println((end-start)+"ms");
return result;
}
/**
* 批量添加
* @param: []
* @return: org.elasticsearch.rest.RestStatus
* @auther: LHL
* @date: 2018/10/15 14:10
*/
@PostMapping("addBulk")
public void addBulk(){
List<Book> bookList = Stream
.iterate(1, i -> i + 1)
.limit(1000000L)
.parallel()
.map(integer -> new Book(String.valueOf(integer), "书名" + integer, "信息" + integer, Double.valueOf(integer), new Date()))
.collect(Collectors.toList());
long start = System.currentTimeMillis();
ElasticsearchUtil.bulkAddDocument(indexName, esType, bookList);
long end = System.currentTimeMillis();
System.out.println((end-start)/1000+"s");
System.out.println((end-start)+"ms");
}
/**
* 批量添加
* @param: []
* @return: void
* @auther: LHL
* @date: 2018/10/16 13:55
*/
@PostMapping("/bulkProcessorAdd")
public void bulkProcessorAdd (){
List<Book> bookList = Stream
.iterate(1, i -> i + 1)
.limit(1000000L)
.parallel()
.map(integer -> new Book(String.valueOf(integer), "书名" + integer, "信息" + integer, Double.valueOf(integer), new Date()))
.collect(Collectors.toList());
long start = System.currentTimeMillis();
ElasticsearchUtil.bulkProcessorAdd(indexName, esType, bookList);
long end = System.currentTimeMillis();
System.out.println((end-start)/1000+"s");
System.out.println((end-start)+"ms");
}
/**
* 批量删除
* @param: []
* @return: org.elasticsearch.rest.RestStatus
* @auther: LHL
* @date: 2018/10/15 14:18
*/
@DeleteMapping("deleteBulk")
public RestStatus deleteBulk(){
List<String> idsList = Stream
.iterate(1, i -> i + 1)
.limit(1000000L)
.parallel()
.map(integer -> String.valueOf(integer))
.collect(Collectors.toList());
long start = System.currentTimeMillis();
BulkResponse bulkItemResponses = ElasticsearchUtil.bulkDeleteDocument(indexName, esType, idsList);
if (bulkItemResponses.hasFailures()){
System.out.println(bulkItemResponses.buildFailureMessage());
}
long end = System.currentTimeMillis();
System.out.println((end-start)/1000+"s");
System.out.println((end-start)+"ms");
return bulkItemResponses.status();
}
/**
* 获取所有
* @param: []
* @return: java.lang.String
* @auther: LHL
* @date: 2018/10/15 15:03
*/
@GetMapping("/getAll")
public List<String> getAll(){
return ElasticsearchUtil.searchAll(indexName);
}
}
Spring Boot 启动报错
- 错误信息
[ main] o.a.catalina.core.AprLifecycleListener : An incompatible version [1.2.7] of the APR based Apache Tomcat Native library is installed, while Tomcat requires version [1.2.14]
- 在 tomcat仓库上下载tomcat-native-1.2.14-ocsp-win32-bin.zip
- 解压zip文件
- 复制
- 32 位 tomcat-native-1.2.14-win32-bin\bin下的【tcnative-1.dll 、tcnative-1-src.pdb 】复制放在C:/Windows/system32/下
- 64 位 tomcat-native-1.2.14-win32-bin\bin\x64下的【tcnative-1.dll 、tcnative-1-src.pdb 】复制放在C:/Windows/system32/下
GitHub代码地址