Es分页

Es 分页

package com.kyexpress.trace.analyzer.test;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.HashMap;
import java.util.Map;

import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;

import com.kyexpress.trace.analyzer.elasticsearch.ElasticsearchClusterProperties;
import com.kyexpress.trace.analyzer.utils.JsonTools;

public class MapMain {

public static void searchEs(RestHighLevelClient restHighLevelClient, int i) {
	try {
		SearchRequest searchRequest = new SearchRequest("map_basic_region_v13");
		searchRequest.types("region");
		SearchSourceBuilder srb = new SearchSourceBuilder();
		// 继续拼接where条件
		int size = 50;
		srb.from((i - 1) * size).size(size);
		BoolQueryBuilder bq = QueryBuilders.boolQuery();
		srb.sort("createAt", SortOrder.ASC);
		srb.query(bq);
		searchRequest.source(srb);
		try {
			SearchResponse response = restHighLevelClient.search(searchRequest);
			SearchHits hits = response.getHits();
			SearchHit[] searchHists = hits.getHits();
			BulkRequest bulkRequest = new BulkRequest();
			for (SearchHit hit : searchHists) {
				Map hitMap = hit.getSourceAsMap();
				Map location = new HashMap<>();
				location.put("lat",
						new BigDecimal(hitMap.get("centerLat") == null ? "0" : hitMap.get("centerLat").toString()));
				location.put("lon",
						new BigDecimal(hitMap.get("centerLng") == null ? "0" : hitMap.get("centerLng").toString()));
				hitMap.put("location", location);
				IndexRequest indexRequest = null;
				indexRequest = new IndexRequest("map_basic_region_v13_20200629");
				indexRequest.source(JsonTools.writeValueAsString(hitMap).toString(), XContentType.JSON);
				indexRequest.type("region");
				bulkRequest.add(indexRequest);
			}
			restHighLevelClient.bulk(bulkRequest);
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	} catch (Exception ex) {
	}
}

public static void scrollEsData(RestHighLevelClient restHighLevelClient) {
	final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
	SearchRequest searchRequest = new SearchRequest("map_basic_region_v13"); // 新建索引搜索请求
	searchRequest.scroll(scroll);
	SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
	BoolQueryBuilder bq = QueryBuilders.boolQuery();
	searchSourceBuilder.sort("createAt", SortOrder.ASC);
	searchSourceBuilder.query(bq);
	searchSourceBuilder.size(100); // 设定每次返回多少条数据
	// searchSourceBuilder.fetchSource(new String[] { "nid" }, null);//
	// 设置返回字段和排除字段
	searchRequest.source(searchSourceBuilder);
	SearchResponse searchResponse = null;
	try {
		searchResponse = restHighLevelClient.search(searchRequest);
		SearchHit[] searchHits = searchResponse.getHits().getHits();
		BulkRequest bulkRequest = new BulkRequest();
		for (SearchHit searchHit : searchHits) {
			Map hitMap = searchHit.getSourceAsMap();
			Map location = new HashMap<>();
			location.put("lat",
					new BigDecimal(hitMap.get("centerLat") == null ? "0" : hitMap.get("centerLat").toString()));
			location.put("lon",
					new BigDecimal(hitMap.get("centerLng") == null ? "0" : hitMap.get("centerLng").toString()));
			hitMap.put("location", location);
			IndexRequest indexRequest = null;
			indexRequest = new IndexRequest("map_basic_region_v13_20200628");
			indexRequest.source(JsonTools.writeValueAsString(hitMap).toString(), XContentType.JSON);
			indexRequest.type("region");
			bulkRequest.add(indexRequest);
		}
		restHighLevelClient.bulk(bulkRequest);

		String scrollId = searchResponse.getScrollId();
		while (searchHits != null && searchHits.length > 0) {
			BulkRequest bulk = new BulkRequest();
			SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
			scrollRequest.scroll(scroll);
			try {
				searchResponse = restHighLevelClient.searchScroll(scrollRequest);
			} catch (Exception e) {
				e.printStackTrace();
			}
			scrollId = searchResponse.getScrollId();
			searchHits = searchResponse.getHits().getHits();
			if (searchHits != null && searchHits.length > 0) {
				for (SearchHit searchHit : searchHits) {
					Map hitMap = searchHit.getSourceAsMap();
					Map location = new HashMap<>();
					location.put("lat", new BigDecimal(
							hitMap.get("centerLat") == null ? "0" : hitMap.get("centerLat").toString()));
					location.put("lon", new BigDecimal(
							hitMap.get("centerLng") == null ? "0" : hitMap.get("centerLng").toString()));
					hitMap.put("location", location);
					IndexRequest indexRequest = null;
					indexRequest = new IndexRequest("map_basic_region_v13_20200628");
					indexRequest.source(JsonTools.writeValueAsString(hitMap).toString(), XContentType.JSON);
					indexRequest.type("region");
					bulk.add(indexRequest);
				}
				restHighLevelClient.bulk(bulkRequest);
			}
		}
		ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
		clearScrollRequest.addScrollId(scrollId);// 也可以选择setScrollIds()将多个scrollId一起使用
		ClearScrollResponse clearScrollResponse = null;
		try {
			clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest);
		} catch (IOException e) {
			e.printStackTrace();
		}
		boolean succeeded = clearScrollResponse.isSucceeded();
		System.out.println("succeeded:" + succeeded);
	} catch (Exception e) {

	}

}

public static void main(String[] args) throws Exception {
	ElasticsearchClient client = new ElasticsearchClient(getProperties());
	RestHighLevelClient restHighLevelClient = client.buildClient();
	// for (int i = 1; i <= 74171; i++) {
	// RestHighLevelClient restHighLevelClient = client.buildClient();
	// searchEs(restHighLevelClient, i);
	// System.out.println("目前执行第" + i + "页");
	// restHighLevelClient.close();
	// }
	scrollEsData(restHighLevelClient);
	System.out.println("结束");
}

private static ElasticsearchClusterProperties getProperties() {
	ElasticsearchClusterProperties properties = new ElasticsearchClusterProperties();
	properties.setClusterNodes("10.121.37.36:9200,10.121.37.37:9200,10.121.37.38:9200");
	Map pp = new HashMap<>();
	// pp.put("user", "elastic")8
	// pp.put("password", "elastic@uat@123456");
	properties.setProperties(pp);
	return properties;
}

}