Elasticsearch批量修改api
Elasticsearch批量修改文档：先根据条件获取去全量文档（获取时如果数据量较大可以进行分页处理，然后进行批量处理；数据量较小就不用分页和批量操作了，边循环边根据当前id修改）
相关代码：
Tool类：
package com.test.util;
import com.demo.common.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import java.net.InetAddress;
import java.util.Properties;
/**
* @description: es tool
* @author: fengze
* @create: 2019-01-07 18:00
**/
public class Tool {
private static Logger logger = Logger.getLogger(Tool.class);
//用集群名字，集群节点地址构建es client
public static Client CLIENT = null;//保证单例，单例，单例！！！ 程序中千万不要new 多个client
//获取项目根目录下的common文件，不同环境文件属性值有差异
public static Properties localProp =
PropertiesUtil.loadProperties("common.properties");
static {
//获取配置文件中定义的es链接属性
try {
CLIENT = getClient(localProp.getProperty("demo.es.config.clusterName"),localProp.getProperty("demo.es.config.clusterNodes"));
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
}
/**
* 创建ES客户端，请自己保证单例，单例，单例！！！千万不要new多个客户端
* @param clusterName 集群名
* @param ipPortList 支持多个ip，port,以逗号和分号分隔。格式 "ip:port,ip:port,ip:port"
* 输入示例 "192.168.200.190:9203,192.168.200.191:9203"
* @return
*/
public static synchronized Client getClient(String clusterName, String ipPortList){
try{
if(clusterName == null || clusterName.isEmpty() || ipPortList == null || ipPortList.isEmpty()){
throw new Exception("集群名，连接地址没有设置");
}
//设置集群的名字
Settings settings = Settings.settingsBuilder()
.put("cluster.name", clusterName)
.put("client.transport.sniff", false)
.build();
//创建集群client并添加集群节点地址
//创建集群client并添加集群节点地址,可以addTransportAddress()多个ip和端口，增加连接的稳定性。
TransportClient client = TransportClient.builder().settings(settings).build();
String[] ipPortArr = ipPortList.split(",");//逗号分隔
for (String ipPort : ipPortArr){
String[] ip_port = ipPort.split(":");//冒号分隔
if(ip_port.length == 2){
client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip_port[0]), Integer.parseInt(ip_port[1])));
}
}
return client;
}catch (Exception e){
logger.error(e.getMessage(),e);
return null;
}
}
}
PropertiesUtil 类：
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
package com.demo.common.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PropertiesUtil {
private static Logger logger = LoggerFactory.getLogger(PropertiesUtil.class);
public PropertiesUtil() {
}
public static Properties loadProperties(String propertyFileName) {
Properties prop = new Properties();
InputStream in = null;
try {
ClassLoader loder = Thread.currentThread().getContextClassLoader();
URL propertyUrl = loder.getResource(propertyFileName);
if (propertyUrl != null) {
in = loder.getResourceAsStream(propertyFileName);
if (in != null) {
prop.load(in);
}
}
} catch (IOException var13) {
logger.error("load {} error!", propertyFileName);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException var12) {
logger.error("close {} error!", propertyFileName);
}
}
}
return prop;
}
public static Properties loadFileProperties(String propertyFileName) {
Properties prop = new Properties();
FileInputStream in = null;
try {
URL propertyUrl = (new File(propertyFileName)).toURI().toURL();
in = new FileInputStream(propertyUrl.getPath());
if (in != null) {
prop.load(in);
}
} catch (IOException var12) {
logger.error("load {} error!", propertyFileName);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException var11) {
logger.error("close {} error!", propertyFileName);
}
}
}
return prop;
}
public static String getString(Properties prop, String key) {
return prop.getProperty(key);
}
public static int getInt(Properties prop, String key) {
return Integer.parseInt(getString(prop, key));
}
public static boolean getBoolean(Properties prop, String key) {
return Boolean.valueOf(getString(prop, key)).booleanValue();
}
}
具体代码逻辑：
//批量修改
//批量修改
@RequestMapping("/es/updateEs2")
@ResponseBody
public Result updateEs2(String type/*mapping*/, String dateFrom/*开始时间*/, String dateTo/*结束时间*/, int size/*每页条数*/) {
int tail = 0;
boolean flag = false;
logger.info("es_update_param,type:{},dateF:{},dateT:{}", type, dateFrom, dateTo);
SearchResponse searchResponse = queryAllByTime2Update("eb_monitor", type, dateFrom, dateTo, 0, 0);
int total = (int) searchResponse.getHits().getTotalHits();
int total_num = total % size == 0 ? total / size : total / size + 1;
logger.info("es_update_result_0,searchResponse:{},total:{},total_num:{}", JSON.toJSONString(searchResponse), total, total_num);
for (int i = 1; i <= total_num; i++) {
searchResponse = queryAllByTime2Update("eb_monitor", type, dateFrom, dateTo, (i - 1) * size, size);
SearchHit[] hits = searchResponse.getHits().hits();
BulkRequestBuilder bulk = Tool.CLIENT.prepareBulk();
logger.info("当前第{}页,size为{}", i, size);
for (SearchHit h : hits) {
Map source = h.getSource();
String id = h.getId();
//如果有该字段，且内容不为空，则不进行修改
if (source.get(("channel")) == null) {
UpdateRequestBuilder data = updateDataById2(h.getIndex(), h.getType(), id);
bulk.add(data);
flag = true;
tail++;
}
}
if (flag) {
BulkResponse responses = bulk.get();
if (responses.hasFailures()) {
System.out.println("failures..............:" + responses.buildFailureMessage());
} else {
logger.info("当前修改条数:{}", tail);
}
}
flag = false;
}
return ResultHelper.setResultData("success");
}
private UpdateRequestBuilder updateDataById2(String index, String type, String id) {
UpdateRequestBuilder builder = Tool.CLIENT.prepareUpdate(index, type, id).setDoc(
"{\"success\":1,\"channel\":1,\"rp_id\":\"0\",\"expansion1\":0,\"expansion2\":0" +
",\"expansion3\":0,\"expansion4\":0,\"expansion5\":0}");
return builder;
}
private SearchResponse queryAllByTime2Update(String indexName, String typeName, String dateFrom, String dateTo, Integer fro, Integer size) {
RangeQueryBuilder time = QueryBuilders.rangeQuery("create_time");
if (Objects.nonNull(dateFrom)) {
time.gt(dateFrom);
}
if (Objects.nonNull(dateTo)) {
time.lt(dateTo);
time.format("yyyy-MM-dd HH:mm:ss");
}
SearchRequestBuilder srb = Tool.CLIENT.prepareSearch(indexName).setTypes(typeName).setQuery(time);
SearchResponse sResponse;
if (size != 0) {
sResponse = srb.setFrom(fro).setSize(size).addSort("create_time", SortOrder.DESC).get();
} else {
sResponse = srb.addSort("create_time", SortOrder.DESC).get();
}
return sResponse;
} //单条修改：
@RequestMapping("/es/updateEs2")
@ResponseBody
public Result updateEs2(String type, String dateFrom, String dateTo, int size) {
logger.info("es_update_param,type:{},dateF:{},dateT:{}", type, dateFrom, dateTo);
SearchResponse searchResponse = queryAllByTime2Update("eb_monitor", type, dateFrom, dateTo, 0, 0);
int total = (int) searchResponse.getHits().getTotalHits();
int total_num = total % size == 0 ? total / size : total / size + 1;
logger.info("es_update_result_0,searchResponse:{},total:{},total_num:{}", JSON.toJSONString(searchResponse), total, total_num);
for (int i = 1; i <= total_num; i++) {
searchResponse = queryAllByTime2Update("eb_monitor", type, dateFrom, dateTo, (i - 1) * size, size);
SearchHit[] hits = searchResponse.getHits().hits();
logger.info("当前第{}页,size为{}", i, size);
for (SearchHit h : hits) {
Map source = h.getSource();
String id = h.getId();
//如果有该字段，且内容不为空，则不进行修改
if (source.get(("channel")) == null) {
updateDataById2(h.getIndex(), h.getType(), id);
}
}
}
return ResultHelper.setResultData("success");
}
private void updateDataById2(String index, String type, String id) {
UpdateRequestBuilder builder = Tool.CLIENT.prepareUpdate(index, type, id).setDoc(
"{\"channel\":1}").get();
GetResponse getResponse = Tool.CLIENT.prepareGet(index, type, id).get();
logger.info("修改后：" + getResponse.getSourceAsString());
}
private SearchResponse queryAllByTime2Update(String indexName, String typeName, String dateFrom, String dateTo, Integer fro, Integer size) {
RangeQueryBuilder time = QueryBuilders.rangeQuery("create_time");
if (Objects.nonNull(dateFrom)) {
time.gt(dateFrom);
}
if (Objects.nonNull(dateTo)) {
time.lt(dateTo);
time.format("yyyy-MM-dd HH:mm:ss");
}
SearchRequestBuilder srb = Tool.CLIENT.prepareSearch(indexName).setTypes(typeName).setQuery(time);
SearchResponse sResponse;
if (size != 0) {
sResponse = srb.setFrom(fro).setSize(size).addSort("create_time", SortOrder.DESC).get();
} else {
sResponse = srb.addSort("create_time", SortOrder.DESC).get();
}
return sResponse;
}