elasticsearch scroll查询
elasticsearch scroll查询
一般的分页查询请求：from、size、n（主分片数）
请求会分发到对应主分片或者副本分片上，分片向协调节点返回(from+size)条数据，n个分片共返回数据(from+size)*n条数据；
协调节点对数据进行排序，然后向客户端返回from到from+size之间的数据
scroll查询：如果要查询100页数据，每页数据100条，则需重复上述过程100次，且随着from的增大，查询效率会越来越慢，使用scroll查询只要一次查询，查询过程：
每个分片返回10000条数据，协调节点共接收数据10000*n条；
协调节点对数据排序后，返回前10000条数据给客户端
********************
相关类与接口
ElasticsearchRestTemplate：操作类
public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate implements ElasticsearchOperations, EsClient, ApplicationContextAware {
public ScrolledPage startScroll(long scrollTimeInMillis, SearchQuery searchQuery, Class clazz) {
public ScrolledPage startScroll(long scrollTimeInMillis, CriteriaQuery criteriaQuery, Class clazz) {
public ScrolledPage startScroll(long scrollTimeInMillis, SearchQuery searchQuery, Class clazz, SearchResultMapper mapper) {
public ScrolledPage startScroll(long scrollTimeInMillis, CriteriaQuery criteriaQuery, Class clazz, SearchResultMapper mapper) {
public ScrolledPage continueScroll(@Nullable String scrollId, long scrollTimeInMillis, Class clazz) {
public ScrolledPage continueScroll(@Nullable String scrollId, long scrollTimeInMillis, Class clazz, SearchResultMapper mapper) {
public void clearScroll(String scrollId) {
ScrolledPage：scroll分页
public interface ScrolledPage extends Page {
String getScrollId();
}
Page：分页
public interface Page extends Slice {
static Page empty() {
return empty(Pageable.unpaged());
}
static Page empty(Pageable pageable) {
return new PageImpl(Collections.emptyList(), pageable, 0L);
}
int getTotalPages();
long getTotalElements();
Page map(Function var1);
}
Slice
public interface Slice extends Streamable {
int getNumber();
int getSize();
int getNumberOfElements();
List getContent();
boolean hasContent();
Sort getSort();
boolean isFirst();
boolean isLast();
boolean hasNext();
boolean hasPrevious();
default Pageable getPageable() {
return PageRequest.of(this.getNumber(), this.getSize(), this.getSort());
}
Pageable nextPageable();
Pageable previousPageable();
Slice map(Function var1);
default Pageable nextOrLastPageable() {
return this.hasNext() ? this.nextPageable() : this.getPageable();
}
default Pageable previousOrFirstPageable() {
return this.hasPrevious() ? this.previousPageable() : this.getPageable();
}
}
********************
示例
***************
pojo 层
User
@Data
@Document(indexName = "user")
public class User {
@Id
private Integer id;
@Field(type = FieldType.Keyword)
private String name;
@Field(type = FieldType.Integer)
private Integer age;
}
***************
controller 层
UserController
@RestController
public class UserController {
@Resource
private UserRepository userRepository;
@Resource
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@RequestMapping("/save4")
public String save(){
for (int i=0;i<1000;i++){
User user=new User();
user.setId(i);
user.setName("瓜田李下 "+i);
user.setAge(i%20+15);
userRepository.save(user);
}
return "success";
}
@RequestMapping("/get8")
public List get(){
QueryBuilder queryBuilder= QueryBuilders.rangeQuery("age").gte(32);
NativeSearchQuery nativeSearchQuery=new NativeSearchQueryBuilder()
.withQuery(queryBuilder)
.withPageable(PageRequest.of(0,20))
.build();
long scrollTimeout = 10000;
ScrolledPage scrolledPage=elasticsearchRestTemplate.startScroll(scrollTimeout,nativeSearchQuery,User.class);
System.out.println("总记录数为："+scrolledPage.getTotalElements());
List resultList=new ArrayList<>();
while (scrolledPage.hasContent()){
List list=scrolledPage.getContent();
resultList.addAll(list);
System.out.println(list);
System.out.println("当前页的条数为："+list.size()+"\n");
scrolledPage=elasticsearchRestTemplate.continueScroll(scrolledPage.getScrollId(), scrollTimeout,User.class);
}
return resultList;
}
}
********************
控制台输出
/get8
总记录数为：150
[User(id=17, name=瓜田李下 17, age=32), User(id=19, name=瓜田李下 19, age=34), User(id=37, name=瓜田李下 37, age=32), User(id=177, name=瓜田李下 177, age=32), User(id=318, name=瓜田李下 318, age=33), User(id=357, name=瓜田李下 357, age=32), User(id=359, name=瓜田李下 359, age=34), User(id=437, name=瓜田李下 437, age=32), User(id=617, name=瓜田李下 617, age=32), User(id=757, name=瓜田李下 757, age=32), User(id=797, name=瓜田李下 797, age=32), User(id=858, name=瓜田李下 858, age=33), User(id=859, name=瓜田李下 859, age=34), User(id=918, name=瓜田李下 918, age=33), User(id=937, name=瓜田李下 937, age=32), User(id=939, name=瓜田李下 939, age=34), User(id=199, name=瓜田李下 199, age=34), User(id=379, name=瓜田李下 379, age=34), User(id=538, name=瓜田李下 538, age=33), User(id=557, name=瓜田李下 557, age=32)]
当前页的条数为：20
[User(id=579, name=瓜田李下 579, age=34), User(id=738, name=瓜田李下 738, age=33), User(id=817, name=瓜田李下 817, age=32), User(id=818, name=瓜田李下 818, age=33), User(id=819, name=瓜田李下 819, age=34), User(id=879, name=瓜田李下 879, age=34), User(id=957, name=瓜田李下 957, age=32), User(id=978, name=瓜田李下 978, age=33), User(id=38, name=瓜田李下 38, age=33), User(id=99, name=瓜田李下 99, age=34), User(id=178, name=瓜田李下 178, age=33), User(id=257, name=瓜田李下 257, age=32), User(id=377, name=瓜田李下 377, age=32), User(id=499, name=瓜田李下 499, age=34), User(id=519, name=瓜田李下 519, age=34), User(id=559, name=瓜田李下 559, age=34), User(id=618, name=瓜田李下 618, age=33), User(id=619, name=瓜田李下 619, age=34), User(id=697, name=瓜田李下 697, age=32), User(id=717, name=瓜田李下 717, age=32)]
当前页的条数为：20
[User(id=719, name=瓜田李下 719, age=34), User(id=917, name=瓜田李下 917, age=32), User(id=59, name=瓜田李下 59, age=34), User(id=159, name=瓜田李下 159, age=34), User(id=217, name=瓜田李下 217, age=32), User(id=219, name=瓜田李下 219, age=34), User(id=239, name=瓜田李下 239, age=34), User(id=298, name=瓜田李下 298, age=33), User(id=299, name=瓜田李下 299, age=34), User(id=317, name=瓜田李下 317, age=32), User(id=338, name=瓜田李下 338, age=33), User(id=339, name=瓜田李下 339, age=34), User(id=678, name=瓜田李下 678, age=33), User(id=737, name=瓜田李下 737, age=32), User(id=759, name=瓜田李下 759, age=34), User(id=977, name=瓜田李下 977, age=32), User(id=979, name=瓜田李下 979, age=34), User(id=998, name=瓜田李下 998, age=33), User(id=39, name=瓜田李下 39, age=34), User(id=119, name=瓜田李下 119, age=34)]
当前页的条数为：20
[User(id=139, name=瓜田李下 139, age=34), User(id=279, name=瓜田李下 279, age=34), User(id=397, name=瓜田李下 397, age=32), User(id=518, name=瓜田李下 518, age=33), User(id=638, name=瓜田李下 638, age=33), User(id=639, name=瓜田李下 639, age=34), User(id=657, name=瓜田李下 657, age=32), User(id=838, name=瓜田李下 838, age=33), User(id=839, name=瓜田李下 839, age=34), User(id=959, name=瓜田李下 959, age=34), User(id=58, name=瓜田李下 58, age=33), User(id=77, name=瓜田李下 77, age=32), User(id=78, name=瓜田李下 78, age=33), User(id=157, name=瓜田李下 157, age=32), User(id=237, name=瓜田李下 237, age=32), User(id=258, name=瓜田李下 258, age=33), User(id=319, name=瓜田李下 319, age=34), User(id=439, name=瓜田李下 439, age=34), User(id=458, name=瓜田李下 458, age=33), User(id=578, name=瓜田李下 578, age=33)]
当前页的条数为：20
[User(id=658, name=瓜田李下 658, age=33), User(id=698, name=瓜田李下 698, age=33), User(id=739, name=瓜田李下 739, age=34), User(id=799, name=瓜田李下 799, age=34), User(id=877, name=瓜田李下 877, age=32), User(id=999, name=瓜田李下 999, age=34), User(id=57, name=瓜田李下 57, age=32), User(id=137, name=瓜田李下 137, age=32), User(id=259, name=瓜田李下 259, age=34), User(id=378, name=瓜田李下 378, age=33), User(id=457, name=瓜田李下 457, age=32), User(id=459, name=瓜田李下 459, age=34), User(id=677, name=瓜田李下 677, age=32), User(id=718, name=瓜田李下 718, age=33), User(id=899, name=瓜田李下 899, age=34), User(id=919, name=瓜田李下 919, age=34), User(id=117, name=瓜田李下 117, age=32), User(id=118, name=瓜田李下 118, age=33), User(id=97, name=瓜田李下 97, age=32), User(id=218, name=瓜田李下 218, age=33)]
当前页的条数为：20
[User(id=277, name=瓜田李下 277, age=32), User(id=297, name=瓜田李下 297, age=32), User(id=398, name=瓜田李下 398, age=33), User(id=417, name=瓜田李下 417, age=32), User(id=418, name=瓜田李下 418, age=33), User(id=419, name=瓜田李下 419, age=34), User(id=438, name=瓜田李下 438, age=33), User(id=497, name=瓜田李下 497, age=32), User(id=517, name=瓜田李下 517, age=32), User(id=598, name=瓜田李下 598, age=33), User(id=679, name=瓜田李下 679, age=34), User(id=758, name=瓜田李下 758, age=33), User(id=777, name=瓜田李下 777, age=32), User(id=778, name=瓜田李下 778, age=33), User(id=837, name=瓜田李下 837, age=32), User(id=857, name=瓜田李下 857, age=32), User(id=878, name=瓜田李下 878, age=33), User(id=958, name=瓜田李下 958, age=33), User(id=997, name=瓜田李下 997, age=32), User(id=18, name=瓜田李下 18, age=33)]
当前页的条数为：20
[User(id=98, name=瓜田李下 98, age=33), User(id=197, name=瓜田李下 197, age=32), User(id=198, name=瓜田李下 198, age=33), User(id=337, name=瓜田李下 337, age=32), User(id=399, name=瓜田李下 399, age=34), User(id=479, name=瓜田李下 479, age=34), User(id=498, name=瓜田李下 498, age=33), User(id=558, name=瓜田李下 558, age=33), User(id=577, name=瓜田李下 577, age=32), User(id=637, name=瓜田李下 637, age=32), User(id=659, name=瓜田李下 659, age=34), User(id=779, name=瓜田李下 779, age=34), User(id=897, name=瓜田李下 897, age=32), User(id=898, name=瓜田李下 898, age=33), User(id=79, name=瓜田李下 79, age=34), User(id=158, name=瓜田李下 158, age=33), User(id=179, name=瓜田李下 179, age=34), User(id=138, name=瓜田李下 138, age=33), User(id=238, name=瓜田李下 238, age=33), User(id=278, name=瓜田李下 278, age=33)]
当前页的条数为：20
[User(id=358, name=瓜田李下 358, age=33), User(id=477, name=瓜田李下 477, age=32), User(id=478, name=瓜田李下 478, age=33), User(id=537, name=瓜田李下 537, age=32), User(id=539, name=瓜田李下 539, age=34), User(id=597, name=瓜田李下 597, age=32), User(id=599, name=瓜田李下 599, age=34), User(id=699, name=瓜田李下 699, age=34), User(id=798, name=瓜田李下 798, age=33), User(id=938, name=瓜田李下 938, age=33)]
当前页的条数为：10