Elasticsearch 使用java分页查询条数超过1w的解决办法
1.需求：查询es近3个月数据取出并进行重新推送。
问题：原使用java分页查询es。可是到1w条数据后就会自动停止无法查询1w之后的数据。
原查询方法：
/**
* @description: 查询轨迹信息
* @param billTraceSearchVo
* @param page
* @date: 2020/5/28 15:06
* @author: ss
* @return: java.util.List
*/
public List findBillTraceInfoDoc(BillTraceSearchVo billTraceSearchVo, Page page) {
List result = new ArrayList<>();
try {
//查询请求对象
SearchRequestBuilder srb = baseQuery(BillTraceElasticSearchDocument.class);
//查询条件
//billCode in (xx,xx)
BoolQueryBuilder bqb = QueryBuilders.boolQuery();
//ID规则
//(id like 'A%' or id like 'B%')
BoolQueryBuilder bqbId = QueryBuilders.boolQuery();
//billCode in (xx,xx)
//运单批量查询条件
if (billTraceSearchVo.getBillCodeList() != null && !billTraceSearchVo.getBillCodeList().isEmpty()) {
QueryBuilder tqb = QueryBuilders.termsQuery("billCode", billTraceSearchVo.getBillCodeList());
bqb.filter(tqb);
}
//开始结束时间不为空查询
if (billTraceSearchVo.getStartTime()!=null &&billTraceSearchVo.getEndTime()!= null){
bqb.must(QueryBuilders.rangeQuery("createTime").from(billTraceSearchVo.getStartTime().longValue()).to(billTraceSearchVo.getEndTime().longValue()));
}
if (billTraceSearchVo.getIdPrefix() != null && !billTraceSearchVo.getIdPrefix().isEmpty()){
//运单前缀
bqbId.should(QueryBuilders.prefixQuery("id", billTraceSearchVo.getIdPrefix()));//运单前缀
}else {
//扫描
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.SCAN_ID_PREFIX));
//同行扫描
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.PEER_ID_PREFIX));
//签收
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.SIGN_ID_PREFIX));
}
//拼接条件billCode in (xx,xx) and (id like 'A%' or id like 'B%')
bqb.filter(bqbId);
//拼接条件deleteFlag == false
bqb.filter(QueryBuilders.termQuery("deleteFlag", false));
//操作类型不能为空
bqb.filter(QueryBuilders.existsQuery("actionType"));
//快件跟踪过滤PTP系统轨迹
BoolQueryBuilder bqbPtp = QueryBuilders.boolQuery();
bqbPtp.mustNot(QueryBuilders.rangeQuery("actionType").gte(Constants.PTP_ACTION_TYPE_MIN).lte(Constants.PTP_ACTION_TYPE_MAX));
bqb.filter(bqbPtp);
//构建分页信息
buildPage(srb, page);
//设置查询条件
srb.setQuery(bqb);
//按照操作时间排序
srb.addSort("createTime", SortOrder.ASC);
//查询数据
SearchResponse response = srb.get();
//结果解析
result = resultParse(response, BillTraceElasticSearchDocument.class);
} catch (Exception e) {
logger.error("查询轨迹ES异常",e);
}
return result;
}
/**
* @description: ES查询结果解析
* @date:
* @param searchResponse
* @param clazz
* @author: ss
* @return: java.util.List
*/
protected List resultParse(SearchResponse searchResponse, Class clazz) {
List list = new ArrayList<>();
if (null != searchResponse) {
SearchHits sh = searchResponse.getHits();
SearchHit[] shs = sh.getHits();
for (int i = 0; i < shs.length; i++) {
SearchHit hit = shs[i];
String result = hit.getSourceAsString();
if (StringUtil.isNotBlank(result)) {
list.add((DOCUMENT) FastJsonUtil.toObject(result, clazz));
}
}
}
return list;
}
/**
* (非 Javadoc) 获取基础查询信息
*
* Title: baseQuery
*
*
* Description:
*
* @param clazz
* @return
* @see cn.uce.core.elasticsearch.repository.ElasticsearchRepository#(cn.uce.core.elasticsearch.document.AbstractElasticsearchDocument,
* java.lang.Class)
*/
public SearchRequestBuilder baseQuery(Class clazz) {
// 获取文档配置
ElasticsearchDocumentConfig config = ElasticsearchDocumentResolver.getDocumentConfig(clazz);
SearchRequestBuilder grb = transportClient.prepareSearch(config.getIndexName());
grb.setTypes(config.getType());
if(StringUtil.isNotBlank(config.getRouting())){
grb.setRouting(config.getRouting());
}
grb.setFrom(0); // 默认从第一条
grb.setSize(1000); // 默认查1000条
return grb;
}
调用查询
Page page = new Page();
page.setPageSize(100);
BillTraceSearchVo condition = new BillTraceSearchVo();
condition.setStartTime(sTime);
condition.setEndTime(eTime);
//分页查询数据
while (true) {
List tmpBillTraceDocList = billTraceElasticSearchRepository.findBillTraceInfoDoc(condition,page);
if (tmpBillTraceDocList != null) {
for (BillTraceElasticSearchDocument traceElasticSearchDocument : tmpBillTraceDocList) {
List billCode2 = new ArrayList<>();
billCode2.add(traceElasticSearchDocument.getBillCode());
List wetTrackTraceVos = new ArrayList<>();
WetTrackTraceVo wetTrackTraceVo = new WetTrackTraceVo();
wetTrackTraceVo.setActionId(traceElasticSearchDocument.getId());
wetTrackTraceVo.setActionType(traceElasticSearchDocument.getActionType() + "");
wetTrackTraceVo.setBillCode(traceElasticSearchDocument.getBillCode());
wetTrackTraceVo.setPartnerCode(partnerCode);
wetTrackTraceVo.setProductCode(productCode);
List listTrace = new ArrayList<>();
listTrace.add(traceElasticSearchDocument);
wetTrackTraceVos.add(wetTrackTraceVo);
log.info("发送重推数据:"+ FastJsonUtil.toJsonString(wetTrackTraceVos));
traceBiz.sendTraceToDifferentPartnerT(wetTrackTraceVos, listTrace);
}
}
if (tmpBillTraceDocList == null || tmpBillTraceDocList.size() < page.getPageSize()) {
break;
}
page.setCurrentPage(page.getCurrentPage() + 1);
} 无法满足查询需求后。查询网上提供的解决方案 ：
1.由于es查询默认设置返回条数为1w条。修改该条数。方式为
第一种：curl -XPUT http://127.0.0.1:9200/_settings -d '{ "index" : { "max_result_window" : 100000000}}‘
第二种：在config/elasticsearch.yml文件中的最后加上index.max_result_window: 100000000
2.修改查询方式为scrollId游标查询
游标查询文档说明 https://www.elastic.co/guide/cn/elasticsearch/guide/cn/scroll.html
修改方法为：
/**
* @description: scrollId查询轨迹信息
* @param
* @return: java.util.List
*/
public SearchResponse findInfoDocByScroll2(String scrollId) {
SearchResponse response=null;
try {
//查询请求对象
// 获取文档配置
SearchScrollRequestBuilder srb = transportClient.prepareSearchScroll(scrollId);
srb.setScroll(new TimeValue(30000));
//查询数据
response = srb.get();
return response;
} catch (Exception e) {
logger.error("查询轨迹ES异常",e);
}
return null;
}
/**
* @description: 查询轨迹信息
* @param billTraceSearchVo
* @param
* @date: 2019/3/27 15:06
* @author: chentianbin
* @return: java.util.List
*/
public SearchResponse findInfoDocByScroll(BillTraceSearchVo billTraceSearchVo) {
List result = new ArrayList<>();
SearchResponse response=null;
try {
//查询请求对象
SearchRequestBuilder srb=baseQuery(BillTraceElasticSearchDocument.class);
srb.setSize(200);
//查询条件
//billCode in (xx,xx)
BoolQueryBuilder bqb = QueryBuilders.boolQuery();
//ID规则
//(id like 'A%' or id like 'B%')
BoolQueryBuilder bqbId = QueryBuilders.boolQuery();
//billCode in (xx,xx)
//运单批量查询条件
if (billTraceSearchVo.getBillCodeList() != null && !billTraceSearchVo.getBillCodeList().isEmpty()) {
QueryBuilder tqb = QueryBuilders.termsQuery("billCode", billTraceSearchVo.getBillCodeList());
bqb.filter(tqb);
}
//开始结束时间不为空查询
if (billTraceSearchVo.getStartTime()!=null &&billTraceSearchVo.getEndTime()!= null){
bqb.must(QueryBuilders.rangeQuery("createTime").from(billTraceSearchVo.getStartTime().longValue()).to(billTraceSearchVo.getEndTime().longValue()));
}
if (billTraceSearchVo.getIdPrefix() != null && !billTraceSearchVo.getIdPrefix().isEmpty()){
//运单前缀
bqbId.should(QueryBuilders.prefixQuery("id", billTraceSearchVo.getIdPrefix()));//运单前缀
}else {
//扫描
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.SCAN_ID_PREFIX));
//同行扫描
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.PEER_ID_PREFIX));
//签收
bqbId.should(QueryBuilders.prefixQuery("id", IdPrefixConstants.SIGN_ID_PREFIX));
}
//拼接条件billCode in (xx,xx) and (id like 'A%' or id like 'B%')
bqb.filter(bqbId);
//拼接条件deleteFlag == false
bqb.filter(QueryBuilders.termQuery("deleteFlag", false));
//操作类型不能为空
bqb.filter(QueryBuilders.existsQuery("actionType"));
//快件跟踪过滤PTP系统轨迹
BoolQueryBuilder bqbPtp = QueryBuilders.boolQuery();
bqbPtp.mustNot(QueryBuilders.rangeQuery("actionType").gte(Constants.PTP_ACTION_TYPE_MIN).lte(Constants.PTP_ACTION_TYPE_MAX));
bqb.filter(bqbPtp);
srb.setScroll(new TimeValue(30000));
//设置查询条件
srb.setQuery(bqb);
//查询数据
response = srb.get();
return response;
} catch (Exception e) {
logger.error("查询轨迹ES异常",e);
}
return null;
} /**
* 不在使用滚动时清除id
* @param scrollId
* @return
*/
public boolean clearScroll( String scrollId) {
ClearScrollRequestBuilder clearScrollRequestBuilder = transportClient.prepareClearScroll();
clearScrollRequestBuilder.addScrollId(scrollId);
ClearScrollResponse response = clearScrollRequestBuilder.get();
return response.isSucceeded();
}调用
List tmpBillTraceDocList = new ArrayList<>();
//结果解析
if (farst==0) {
SearchResponse searchResponse = billTraceElasticSearchRepository.findInfoDocByScroll(condition);
if (null != searchResponse) {
SearchHits sh = searchResponse.getHits();
SearchHit[] shs = sh.getHits();
for (int i = 0; i < shs.length; i++) {
SearchHit hit = shs[i];
String result = hit.getSourceAsString();
if (StringUtil.isNotBlank(result)) {
tmpBillTraceDocList.add((BillTraceElasticSearchDocument) FastJsonUtil.toObject(result, BillTraceElasticSearchDocument.class));
}
}
scrollId=searchResponse.getScrollId();
} else {
billTraceElasticSearchRepository.clearScroll(scrollId);
break;
}
}else {
SearchResponse searchResponse = billTraceElasticSearchRepository.findInfoDocByScroll2(scrollId);
if (null != searchResponse) {
SearchHits sh = searchResponse.getHits();
SearchHit[] shs = sh.getHits();
for (int i = 0; i < shs.length; i++) {
SearchHit hit = shs[i];
String result = hit.getSourceAsString();
if (StringUtil.isNotBlank(result)) {
tmpBillTraceDocList.add((BillTraceElasticSearchDocument) FastJsonUtil.toObject(result, BillTraceElasticSearchDocument.class));
}
}
scrollId=searchResponse.getScrollId();
} else {
billTraceElasticSearchRepository.clearScroll(scrollId);
break;
}
} 修改后调用查询推送：
推送次数不受查询1w次限制.