说明
maven依赖
官方客户端 https://www.elastic.co/guide/en/elasticsearch/client/java-rest/6.4/index.html
org.elasticsearch elasticsearch 6.5.0 true org.elasticsearch.client elasticsearch-rest-high-level-client 6.5.0 commons-codec commons-codec elasticsearch org.elasticsearch
Util类
配置类
package com.crb.ocms.product.domain.config; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; /** * @Project crb-product-domain * @PackageName com.crb.ocms.product.admin.config * @ClassName ESConfiguration * @Author liqiang * @Date 2019/3/6 5:52 PM * @Description 用于加载es的相关配置 */ @ConfigurationProperties(prefix = "esconfig") @Data public class ESConfiguration { /** * index别名 */ private String aliasName; /** * 索引名字 */ private String indexName; /** * 全量索引的最大处理线程大小 */ private int threadSize; /** * 全量索引并行执行每个线程每次执行数据大小 */ private int treadDataSize; /** * typename */ private String typeName; /** * ESURL */ private String esUrl; /** * es ip */ private String host; /** * es端口 */ private Integer port; /*** * 用于标识是否正在处理的rediskey */ private String redissKey; /** *mappingJson */ private String mappingJson; }
util工具类
package com.crb.ocms.product.service.impl; import com.crb.ocms.product.domain.config.ESConfiguration; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.client.*; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.search.suggest.SuggestionBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.stereotype.Component; import java.io.IOException; import java.util.*; import java.util.regex.Pattern; /** * @Project crb-product-service * @PackageName com.crb.ocms.demo.service.test.impl * @ClassName ESHLRestUtil * @Author liqiang * @Date 2019/2/25 13:45 * @Description es工具类 */ @Slf4j @Component @EnableConfigurationProperties(ESConfiguration.class) public class ESHLRestUtil { private RestHighLevelClient client = null; public RestHighLevelClient getClient() { return client; } public void setClient(RestHighLevelClient client) { this.client = client; } ESConfiguration esConfiguration; @Autowired public ESHLRestUtil(ESConfiguration configuration) { client = new SimpleRestHighLevelClient(RestClient.builder(new HttpHost(configuration.getHost(), configuration.getPort(), "http"))); this.esConfiguration=configuration; } public SimpleRestHighLevelClient getSimpleClient(){ return (SimpleRestHighLevelClient)client; } /** *根据indexname获得index名字 支持通配符匹配 * @param indexName * @return */ public String [] getIndexNames(String indexName) { GetIndexResponse getIndexResponse= null; try { getIndexResponse = getIndexInfo(indexName); } catch (IOException e) { e.printStackTrace(); } return getIndexResponse!=null?getIndexResponse.getIndices():null; } /** * 根据index名字获得index信息 * @param indexName * @return */ public GetIndexResponse getIndexInfo(String indexName) throws IOException { //GetIndexRequest request = new GetIndexRequest().indices(indexName); // try { // GetIndexResponse getIndexResponse =client.indices().get(request, RequestOptions.DEFAULT); // return getIndexResponse; // } catch (IOException e) { // e.printStackTrace(); // } //master_timeout /** * 因为现在是api使用6.5 线上是6.24 master_timeout 使用以下方式 替换掉 */ GetIndexRequest request = new GetIndexRequest().indices(indexName); String[] indices = request.indices() == null ? Strings.EMPTY_ARRAY : request.indices(); String endpoint =indexName; Request httpequest = new Request("GET", endpoint); httpequest.addParameter("ignore_unavailable", "false"); httpequest.addParameter("expand_wildcards", "open"); httpequest.addParameter("allow_no_indices", "true"); org.apache.http.HttpEntity entity = new NStringEntity("", ContentType.APPLICATION_JSON); httpequest.setEntity(entity); Response httpResponse= getClient().getLowLevelClient().performRequest(httpequest); GetIndexResponse getIndexResponse=getSimpleClient().parseGetIndexRespons(httpResponse); return getIndexResponse; } /** * 获得最大index名字 * @param names indexName_number 格式 * @return */ public Integer getMaxIndexNumber(String []names) { if(names==null||names.length<=0){ return null; } ListindexNumber=new ArrayList (); for (String name: names) { String[] arrs=name.split("_"); if(arrs.length<=1){ continue; } indexNumber.add(Integer.valueOf(arrs[1])); } indexNumber.sort(new Comparator () { @Override public int compare(Integer o1, Integer o2) { return o2 - o1; } }); return CollectionUtils.isEmpty(indexNumber)?null:indexNumber.get(0); } /** * 验证索引是否存在 * * @param index 索引名称 * @return * @throws Exception */ public boolean indexExists(String index) throws Exception { GetIndexRequest request = new GetIndexRequest(); request.indices(index); boolean exists = client.indices().exists(request, RequestOptions.DEFAULT); return exists; } /** * 创建index * * @param index * @param indexType * @param properties 结构: {name:{type:text}} {age:{type:integer}} * @return * @throws Exception */ public boolean indexCreate(String index, String indexType, Map properties) throws Exception { if (indexExists(index)) { return true; } CreateIndexRequest request = new CreateIndexRequest(index); request.settings(Settings.builder().put("index.number_of_shards", 3) .put("index.number_of_replicas", 2)); Map jsonMap = new HashMap<>(); Map mapping = new HashMap<>(); mapping.put("properties", properties); jsonMap.put(indexType, mapping); request.mapping(indexType, jsonMap); CreateIndexResponse createIndexResponse = client.indices().create( request,RequestOptions.DEFAULT); boolean acknowledged = createIndexResponse.isAcknowledged(); return acknowledged; } /** * 删除指定索引 * @param indexName * @return * @throws IOException */ public boolean deleteIndex(String indexName) throws IOException { DeleteIndexRequest request = new DeleteIndexRequest(indexName); AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); return deleteIndexResponse.isAcknowledged(); } /** * 创建索引 * @param index 索引名字 * @param settiongs settiongs * @return * @throws Exception * @author lqiang */ public boolean indexCreate(String index, String settiongs ) throws Exception { if (indexExists(index)) { return true; } CreateIndexRequest request = new CreateIndexRequest(index); // request.settings(Settings.builder().put("index.number_of_shards", 3) // .put("index.number_of_replicas", 2)); request.source(settiongs, XContentType.JSON); CreateIndexResponse createIndexResponse = client.indices().create( request,RequestOptions.DEFAULT); boolean acknowledged = createIndexResponse.isAcknowledged(); return acknowledged; } /** * 创建更新文档 * * @param index * @param indexType * @param documentId * @param josonStr * @return * @throws Exception */ public boolean documentCreate(String index, String indexType, String documentId, String josonStr) throws Exception { IndexRequest request = new IndexRequest(index, indexType, documentId); request.source(josonStr, XContentType.JSON); IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); if (indexResponse.getResult() == DocWriteResponse.Result.CREATED || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { return true; } ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); if (shardInfo.getTotal() != shardInfo.getSuccessful()) { return true; } if (shardInfo.getFailed() > 0) { for (ReplicationResponse.ShardInfo.Failure failure : shardInfo .getFailures()) { throw new Exception(failure.reason()); } } return false; } /** * 创建更新文档 * * @param index * @param indexType * @param documentId * @param map * @return * @throws Exception */ public boolean documentCreate(String index, String indexType, String documentId, Map map) throws Exception { IndexRequest request = new IndexRequest(index, indexType, documentId); request.source(map); IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT); if (indexResponse.getResult() == DocWriteResponse.Result.CREATED || indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { return true; } ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); if (shardInfo.getTotal() != shardInfo.getSuccessful()) { return true; } if (shardInfo.getFailed() > 0) { for (ReplicationResponse.ShardInfo.Failure failure : shardInfo .getFailures()) { throw new Exception(failure.reason()); } } return false; } /** * 批量创建更新文档 * * @param index * @param indexType * @param list 建索引时传入_id作为docuemntId * @return * @throws Exception */ public boolean documentCreateBulk(String index, String indexType, List
封装的处理器
工作原理
将指定条件的数据迁移到新索引,然后再新索引上面进行导入(多线程并行导入),导入完毕删除老索引和别名 然后为新索引绑定别名 实现不停机更新
抽象接口
package com.crb.ocms.product.serviceTool; import org.elasticsearch.index.reindex.ReindexRequest; import java.io.IOException; import java.util.concurrent.Future; /** * @Project crb-product-service * @PackageName com.crb.ocms.product.serviceTool * @ClassName EsImportService * @Author liqiang * @Date 2019/3/29 1:28 PM * @Description es导入抽象接口 */ public interface ESImportService { /** * 获得索引迁移条件 * @param reindexRequest * @return * @throws IOException */ public boolean reindex(ReindexRequest reindexRequest) throws IOException; /** * 异步导入 * @return */ public FutureimportAllAsyn(); }
抽象的处理器
使用模板模式将通用代码抽出来
package com.crb.ocms.product.serviceTool; import com.crb.ocms.product.domain.config.ESConfiguration; import com.crb.ocms.product.domain.entity.MdProduct; import com.crb.ocms.product.domain.redisskey.MdEsProductRedisKeyEnum; import com.crb.ocms.product.domain.util.exceptions.OCmsExceptions; import com.crb.ocms.product.service.impl.ESHLRestUtil; import com.hazelcast.util.StringUtil; import lombok.extern.log4j.Log4j2; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; import org.apache.lucene.util.BytesRef; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.ReindexRequest; import org.redisson.api.RedissonClient; import org.springframework.data.domain.PageRequest; import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; /** * @Project crb-product-service * @PackageName com.crb.ocms.product.serviceTool * @ClassName ESAbstrctImpor * @Author liqiang * @Date 2019/3/29 1:35 PM * @Description 抽象的es导入处理器 */ @Log4j2 public abstract class ESAbstractImport implements ESImportService { private RestTemplate restTemplate; private ESConfiguration esConfiguration; private ESHLRestUtil eshlRestUtil; private RedissonClient redissonClient; /** * 当前index名字 */ private String templateIndexName; /** * 之前的index名字 */ private String beforeIndexName; public ESAbstractImport(RestTemplate restTemplate, ESConfiguration esConfiguration,RedissonClient redissonClient){ esConfiguration.setIndexName("cmsproudct"); esConfiguration.setAliasName("cmsproductAliasName"); this.restTemplate=restTemplate; this.esConfiguration=esConfiguration; this.eshlRestUtil=new ESHLRestUtil(esConfiguration); this.redissonClient=redissonClient; } /** * 获得创建索引的mapping抽象方法 * @return */ public abstract String getMapping(); /** * 获得所有迁移requestBody抽象方法 * @return */ public abstract ReindexRequest getReindex(); /** * es索引秦阿姨 * @return */ @Override public boolean reindex(ReindexRequest reindexRequest) throws IOException { // BulkByScrollResponse bulkByScrollResponse= eshlRestUtil.getClient().reindex(reindexRequest,RequestOptions.DEFAULT); /** * 因为现在是api使用6.5 线上是6.24 查询语句多生成zero_terms_query 使用以下方式 替换掉 */ String endpoint ="/_reindex"; Request httpequest = new Request("POST", endpoint); BytesRef source = XContentHelper.toXContent(reindexRequest, XContentType.JSON, false).toBytesRef(); org.apache.http.HttpEntity entity = new NStringEntity(new String(source.bytes).replaceAll("\"zero_terms_query\":\"NONE\","," "), ContentType.APPLICATION_JSON); httpequest.setEntity(entity); Response httpResponse=eshlRestUtil.getClient().getLowLevelClient().performRequest(httpequest); BulkByScrollResponse getIndexResponse=eshlRestUtil.getSimpleClient().parseBulkByScrollResponse(httpResponse); return true; } /** * 导入数据 根据设置的线程大小根据数据大小算出线程数量 通过多线程并行处理 * @return */ public boolean importAll() throws IOException { log.info("正在产品全量导入2:"+esConfiguration.getEsUrl()); if (redissonClient.getMap(esConfiguration.getRedissKey()).isExists()) { throw new OCmsExceptions("全量索引正在导入中...."); } String []indexNames=eshlRestUtil.getIndexNames(esConfiguration.getIndexName()+"*"); if (indexNames!=null&&indexNames.length>0) { Integer number= eshlRestUtil.getMaxIndexNumber(indexNames); beforeIndexName=number==null?esConfiguration.getIndexName():esConfiguration.getIndexName()+"_"+number; templateIndexName=esConfiguration.getIndexName()+"_"+(number==null?1:(number+1)); } else { templateIndexName = esConfiguration.getIndexName(); } boolean isSuccess = true; long count = getCount(); //根据线程处理大小获得页码 int index = (new Double(Math.ceil(count*1.0 / esConfiguration.getTreadDataSize()))).intValue(); if (index <= 0 && count > 0) { index = 1; } //获得最大处理线程大小 int threadSize = index > esConfiguration.getThreadSize() ? esConfiguration.getThreadSize() : index; //导入处理器 ESAbstractImport.ProcessImportIndex processImportIndex = new ESAbstractImport.ProcessImportIndex(esConfiguration.getTreadDataSize(), index, templateIndexName); ThreadPoolExecutor executorService=null; try { //第一次导入直接创建索引 if (beforeIndexName == null) { log.info("开始创建索引"); eshlRestUtil.indexCreate(templateIndexName,getMapping()); log.info("创建索引成功"); } else { log.info("开始创建索引1"); eshlRestUtil.indexCreate(templateIndexName,getMapping()); log.info("创建索引成功1"); log.info("开始创建索引2"); //非第一次导入将子集数据迁移进来 reindex(getReindex()); log.info("开始创建索引2"); } /** 线程池的自定义配置,IO密集型任务. */ executorService = new ThreadPoolExecutor( // 核心线程数 threadSize, // 最大线程数 100, // 存活时间,30s 30, TimeUnit.SECONDS, new ArrayBlockingQueue(150), Executors.defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy()); redissonClient.getMap(esConfiguration.getRedissKey()).put("templateIndexName", templateIndexName); //统一保存异步处理结果 List > processResult = new ArrayList >(); for (int i = 0; i < threadSize; i++) { processResult.add(executorService.submit(processImportIndex)); } for (Future future : processResult) { if (!future.get()) { isSuccess = future.get(); break; } } //表示重建成功 if (isSuccess) { //表示第一次初始化 指定别名 if (StringUtil.isNullOrEmpty(beforeIndexName)) { boolean isFail = false; int i = 0; do { isFail = !eshlRestUtil.aliases(templateIndexName,esConfiguration.getAliasName()); if(i>0) { //休眠 Thread.sleep(i * 1000); } i++; } while (isFail && i < 10); if (isFail) { log.info("开始删除索引2"); eshlRestUtil.deleteIndex(templateIndexName); log.info("开始删除成功"); } } else { boolean isFail = false; int i = 0; do { //重新绑定别名 失败重试十次 isFail = !eshlRestUtil.aliases(beforeIndexName,templateIndexName,esConfiguration.getAliasName()); if(i>0) { //休眠 Thread.sleep(i * 1000); } i++; } while (isFail && i < 10); if (isFail) { //删除备份索引 eshlRestUtil.deleteIndex(templateIndexName); throw new OCmsExceptions("索引重做失败,请重试"); } else { //删除备份索引 eshlRestUtil.deleteIndex(beforeIndexName); } } } else { for (Future future : processResult) { if (!future.get()) { //没有重做成功删除创建的新索引 eshlRestUtil.deleteIndex(templateIndexName); future.isCancelled();//取消其他正在执行的线程 } } } } catch (Exception e) { String uuid = UUID.randomUUID().toString(); log.error(uuid + "\n" + e.getMessage()); eshlRestUtil.deleteIndex(templateIndexName); throw new OCmsExceptions(" 异常编码" + uuid + "\n" + e.getMessage()); } finally { if (executorService != null) { executorService.shutdown(); } redissonClient.getBucket(esConfiguration.getRedissKey()).delete(); } return isSuccess; } /** * 异步的导入方法 * @return */ @Override public Future importAllAsyn() { ExecutorService executorService=new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue ()); return executorService.submit(new Callable () { @Override public Boolean call() throws Exception { return importAll(); } }); } /** * 抽象的处理导入方法 * @param pageRequest * @return * @throws IOException */ public abstract boolean processImport(PageRequest pageRequest) throws IOException; public abstract Long getCount(); /** * 用于处理批量索引导入 */ class ProcessImportIndex implements Callable { //数据总页数 private int pageCount; //总条数 private int pageSize; private String indexName; private AtomicLong currentIndex = new AtomicLong(); //当前处理页数 AtomicInteger currentPage = new AtomicInteger(); public ProcessImportIndex(int pageSize, int pageCount, String indexName) { this.pageSize = pageSize; this.pageCount = pageCount; this.indexName = indexName; } /** * Computes a result, or throws an exception if unable to do so. * * @return computed result * @throws Exception if unable to compute a result */ @Override public Boolean call() throws Exception { try { return process(); } catch (Exception e) { e.printStackTrace(); return false; } } public boolean process() throws IOException { int page = currentPage.getAndIncrement(); redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("pageCount", pageCount); while (page < pageCount) { redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("currentPage", page+1); processImport(new PageRequest(page, pageSize)); page = currentPage.getAndIncrement(); } return true; } public AtomicInteger getCurrentPage() { return currentPage; } public void setCurrentPage(AtomicInteger currentPage) { this.currentPage = currentPage; } } public String getTemplateIndexName() { return templateIndexName; } public void setTemplateIndexName(String templateIndexName) { this.templateIndexName = templateIndexName; } public interface BulkCallback { public void process(MdProduct mdProduct); } public RestTemplate getRestTemplate() { return restTemplate; } public void setRestTemplate(RestTemplate restTemplate) { this.restTemplate = restTemplate; } public ESConfiguration getEsConfiguration() { return esConfiguration; } public void setEsConfiguration(ESConfiguration esConfiguration) { this.esConfiguration = esConfiguration; } public ESHLRestUtil getEshlRestUtil() { return eshlRestUtil; } public void setEshlRestUtil(ESHLRestUtil eshlRestUtil) { this.eshlRestUtil = eshlRestUtil; } public RedissonClient getRedissonClient() { return redissonClient; } public void setRedissonClient(RedissonClient redissonClient) { this.redissonClient = redissonClient; } public String getBeforeIndexName() { return beforeIndexName; } public void setBeforeIndexName(String beforeIndexName) { this.beforeIndexName = beforeIndexName; } }
使用例子
package com.crb.ocms.product.serviceTool; import com.crb.ocms.product.domain.config.ESConfiguration; import com.crb.ocms.product.domain.entity.MdProduct; import com.crb.ocms.product.domain.repository.MdProductRepository; import com.crb.ocms.product.domain.vo.req.FindProductCharacterStocksVo; import com.crb.ocms.product.domain.vo.resp.ProductCharacterStocksVo; import com.crb.ocms.product.service.MdProductService; import com.crb.ocms.product.service.feign.IcProductStoreAccountService; import com.crb.ocms.product.service.impl.MdESProductServiceImpl; import lombok.extern.log4j.Log4j2; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.ReindexRequest; import org.redisson.api.RedissonClient; import org.springframework.data.domain.Example; import org.springframework.data.domain.ExampleMatcher; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; /** * @Project crb-product-service * @PackageName com.crb.ocms.product.serviceTool * @ClassName ProductStockImport * @Author liqiang * @Date 2019/3/29 2:15 PM * @Description 处理库存全量索引导入 */ @Log4j2 @Deprecated public class ProductStockImport extends ProductIndexImport { MdProductRepository mdProductRepository; IcProductStoreAccountService icProductStoreAccountService; String regionCode; public ProductStockImport(RestTemplate restTemplate, ESConfiguration esConfiguration, RedissonClient redissonClient, MdProductRepository mdProductRepository, IcProductStoreAccountService icProductStoreAccountService, String regionCode) { super(restTemplate, esConfiguration, redissonClient); this.mdProductRepository=mdProductRepository; this.regionCode=regionCode; this.icProductStoreAccountService=icProductStoreAccountService; } /** * 索引迁移条件。将指定条件数据迁移到新的索引 * @return */ @Override public ReindexRequest getReindex() { ReindexRequest reindexRequest=new ReindexRequest(); reindexRequest.setSourceIndices(getBeforeIndexName()); reindexRequest.setSourceQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.matchPhraseQuery("info","ic_product_store_account"))); reindexRequest.setDestIndex(getTemplateIndexName()); return reindexRequest; } /** * 已经迁移到经销权 先禁用 * @param pageRequest * @return * @throws IOException */ @Deprecated @Override public boolean processImport(PageRequest pageRequest) throws IOException { MdProduct mdProduct = new MdProduct(); mdProduct.setRegionCode(regionCode); ExampleMatcher exampleMatcher = ExampleMatcher.matching().withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id");; Example example=Example.of(mdProduct,exampleMatcher); Pageresult = mdProductRepository.findAll(example,pageRequest); for (MdProduct md:result){ if (md.getFullPalletSaleFlag()==null){ md.setFullPalletSaleFlag(0); } } List productIds=result.getContent().stream().map(MdProduct::getMdProductId).collect(Collectors.toList()); FindProductCharacterStocksVo findProductCharacterStocksVo=new FindProductCharacterStocksVo(); findProductCharacterStocksVo.setProductIds(productIds); findProductCharacterStocksVo.setRegionCode(regionCode); List productCharacterStocksVoList= icProductStoreAccountService.findProductCharacterStocks(findProductCharacterStocksVo); BulkRequest bulkRequest=new BulkRequest(); for (ProductCharacterStocksVo productCharacterStocksVo: productCharacterStocksVoList) { if(productCharacterStocksVo.getCharacterId().longValue()==-1L){ continue; } XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field("productId",productCharacterStocksVo.getProductId()); builder.field("factoryId",productCharacterStocksVo.getFactoryId()); builder.field("characterId",productCharacterStocksVo.getCharacterId()); builder.field("sumCount",productCharacterStocksVo.getSumCount()); builder.field("regionCode",regionCode); builder.field("info").startObject() .field("name","ic_product_store_account") .field("parent",productCharacterStocksVo.getProductId()+"_"+regionCode) .endObject(); builder.endObject(); String indexId="ic_product_store_account_"+productCharacterStocksVo.getProductId()+"_"+productCharacterStocksVo.getCharacterId()+"_"+productCharacterStocksVo.getFactoryId(); IndexRequest indexRequest=new IndexRequest(getTemplateIndexName(),getEsConfiguration().getTypeName(), indexId); indexRequest.source(builder); indexRequest.routing(productCharacterStocksVo.getProductId()+"_"+regionCode); bulkRequest.add(indexRequest); } log.info(bulkRequest.getDescription()); getEshlRestUtil().getClient().bulk(bulkRequest, RequestOptions.DEFAULT); return true; } /** * 数据总条数 * @return */ @Override public Long getCount() { MdProduct mdProduct = new MdProduct(); mdProduct.setRegionCode(regionCode); ExampleMatcher exampleMatcher = ExampleMatcher.matching(). withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id"); Example example=Example.of(mdProduct,exampleMatcher); return mdProductRepository.count(example); } }
processImport就只用关系查询出指定分页的数据 然后导入到es