导入导出elasticsearch索引数据
1.方式一
dump插件
在elasticsearch5.x中
1.1 elasticsearch-dump安装
1) yum install epel-release
2) yum install nodejs
3) yum install npm
4) npm install elasticdump
5) cd node_modules/elasticdump/bin 后便可以执行操作。1.2 elasticsearch-dump使用
mapping：
./elasticdump --input=http://10.10.10.215:8400/~mdap-china-city_ihl --output=http://10.10.10.138:9200/~mdap-china-city_ihl --type=mapping
data:
2.方式二
代码方式（java 或 python）
java代码：
package com.andong.es.util;
import java.io.File;
import java.io.FileOutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFCellStyle;
import org.apache.poi.hssf.usermodel.HSSFFont;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
/**
* ESUtils
*
* @author andong
*
*/
public class ESUtils
{
public final static DateFormat esdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
private final static Logger logger = LoggerFactory.getLogger(ESUtils.class);
/** ES host */
private static String host = "10.10.10.215";
/** ES port */
private static int port = 8500;
/** ES clusterName */
private static String clusterName = "imdap-215";
/** ES client */
private static Client client = new TransportClient(ImmutableSettings.settingsBuilder()
.put("cluster.name", clusterName).build())
.addTransportAddress(new InetSocketTransportAddress(host, port));
public static Client getClient()
{
return client;
}
/**
* 判断是否存在 类型
*
* @param index
* 索引
* @param types
* 类型
* @return
*/
public static boolean isTypeExists(String index, String... types)
{
IndicesAdminClient indexClient = client.admin().indices();
TypesExistsRequestBuilder typeExist = null;
if (index == null || index.equals(""))
{
typeExist = indexClient.prepareTypesExists();
}
else
{
typeExist = indexClient.prepareTypesExists(index);
}
TypesExistsRequestBuilder typeExists = typeExist.setTypes(types);
TypesExistsResponse response = typeExists.get();
return response.isExists();
}
/**
* 判断是否存在 索引
*
* @param indicese
* 索引
* @return
*/
public static boolean isIndexExists(String... indicese)
{
IndicesExistsRequestBuilder indexExists = client.admin().indices().prepareExists(indicese);
IndicesExistsResponse response = indexExists.get();
return response.isExists();
}
public static void insertOne(String index,String type,Object data){
Listmain方法中： 导出为excel格式
public static void main(String[] args) throws Exception
{
ESUtils esUtils = new ESUtils();
boolean typeExists = esUtils.isTypeExists("~mdap-china-city_ihl", "province");
if(typeExists) {
String[] indexs= {"~mdap-china-city_ihl"};
FilterBuilder filterBuilders = null;
QueryBuilder queryBuilders = null;
LinkedHashMap orders = null;
Pageable search = esUtils.search(indexs, "province", filterBuilders, queryBuilders, orders, 0, 10000);
List> content = search.getContent();
//esUtils.parseEs(content);
// 创建execl 文件
HSSFWorkbook workbook = new HSSFWorkbook();
HSSFSheet sheet = workbook.createSheet();
// 设定默认宽度
sheet.setDefaultColumnWidth(18);
// 数字double样式
HSSFCellStyle contentCellStyle = workbook.createCellStyle();
HSSFFont font = workbook.createFont();
font.setFontName("宋体");
font.setFontHeightInPoints((short) 11);
contentCellStyle.setFont(font);
// 标题样式
HSSFCellStyle headerStyle = (HSSFCellStyle) workbook.createCellStyle();// 创建标题样式
headerStyle.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER); // 设置垂直居中
headerStyle.setAlignment(HSSFCellStyle.ALIGN_CENTER); // 设置水平居中
HSSFFont headerFont = (HSSFFont) workbook.createFont(); // 创建字体样式
headerFont.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD); // 字体加粗
headerFont.setFontName("Times New Roman"); // 设置字体类型
headerFont.setFontHeightInPoints((short) 12); // 设置字体大小
headerStyle.setFont(headerFont); // 为标题样式设置字体样式
HSSFRow row = null;
HSSFCell cell = null;
StringBuffer idBuffer = new StringBuffer("");
LinkedHashMap map = new LinkedHashMap();
row = sheet.createRow(0);
cell = row.createCell(0);
cell.setCellValue("name");
cell = row.createCell(1);
cell.setCellValue("IN_TIME");
cell = row.createCell(2);
cell.setCellValue("latitude");
cell = row.createCell(3);
cell.setCellValue("cid");
cell = row.createCell(4);
cell.setCellValue("id");
cell = row.createCell(5);
cell.setCellValue("longitude");
cell = row.createCell(6);
cell.setCellValue("brief");
int index = 0;
for(int i=0 ; i< content.size();i++) {
index++;
System.out.println(content.get(i));
row = sheet.createRow(index);
cell = row.createCell(0);
cell.setCellValue(content.get(i).get("name").toString());
cell = row.createCell(1);
cell.setCellValue(content.get(i).get("IN_TIME").toString());
cell = row.createCell(2);
cell.setCellValue(content.get(i).get("latitude").toString());
cell = row.createCell(3);
cell.setCellValue(content.get(i).get("cid").toString());
cell = row.createCell(4);
cell.setCellValue(content.get(i).get("id").toString());
cell = row.createCell(5);
cell.setCellValue(content.get(i).get("longitude").toString());
cell = row.createCell(6);
cell.setCellValue(content.get(i).get("brief").toString());
//idBuffer.append(fields.get("cacti_id")+",");
}
FileOutputStream out = new FileOutputStream(new File("D://province.xls"));
workbook.write(out);
out.flush();
}
}
python代码：
# coding: utf-8
import json
import os
import sys
import time
import urllib3
class importEsData():
def __init__(self,url,index,type):
self.url = url+"/"+index+"/"+type
self.index = index
self.type = type
def importData(self):
print("import data begin...")
begin = time.time()
try:
f = open(self.index+"_"+self.type+".json","r")
for line in f:
self.post(line)
finally:
f.close()
print("import data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
def post(self,data):
req = urllib3.request(self.url,data,{"Content-Type":"application/json; charset=UTF-8"})
urllib3.connection(req)
if __name__ == '__main__':
#importEsData("http://10.100.142.60:9200","watchdog","test").importData()
importEsData("http://10.10.10.215:8400/","~mdap-china-city_ihl","mapping").importData()
# coding: utf-8
import json
import os
import sys
import time
import urllib3
class exportEsData():
size = 10000
def __init__(self, url,index,type):
self.url = url+"/"+index+"/"+type+"/_search"
self.index = index
self.type = type
def exportData(self):
print("export data begin...")
begin = time.time()
try:
os.remove(self.index+"_"+self.type+".json")
except:
pass
msg = urllib3.request(self.url).read()
print(msg)
obj = json.loads(msg)
num = obj["hits"]["total"]
start = 0
end = num/self.size+1
while(start