Es bulk批量导入数据(1w+以上)
最近在学习es的理论知识以及实际操作，随时更新~
概要：首先你得有1w条数据的json，然后用java读取json文件导入
一. 创建Json数据
首先我生成1.5w条数据，是为了实践分页查询，用from-size和scroll翻页去实践
生成四个字段，name、age、sex、telephone
代码如下：可直接复制粘贴用
package es;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Random;
/**
* 生成1.5W条json数据
*/
public class JsonGenerator {
public static void main(String[] args) {
int numberOfRecords = 15000;
String outputFile = "user.json";
try (FileWriter writer = new FileWriter(outputFile)) {
for (int i = 0; i < numberOfRecords; i++) {
writer.write(generateJsonRecord());
if (i < numberOfRecords - 1) {
writer.write(",");
} else {
writer.write(" ");
}
}
System.out.println("Generated " + numberOfRecords + " records successfully.");
} catch (IOException e) {
e.printStackTrace();
}
}
private static String generateJsonRecord() {
String name = generateRandomName();
int age = generateRandomAge();
String sex = generateRandomSex();
String telephone = generateRandomTelephone();
return "{\"name\":\"" + name + "\",\"age\":\"" + age + "\",\"sex\":\"" + sex + "\",\"telephone\":\"" + telephone + "\"}";
}
private static String generateRandomName() {
String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
StringBuilder sb = new StringBuilder();
Random random = new Random();
for (int i = 0; i < 5; i++) {
sb.append(characters.charAt(random.nextInt(characters.length())));
}
return sb.toString();
}
private static int generateRandomAge() {
Random random = new Random();
return random.nextInt(65) + 18; // Generate age between 18 and 65
}
private static String generateRandomSex() {
Random random = new Random();
return random.nextBoolean() ? "Male" : "Female";
}
private static String generateRandomTelephone() {
StringBuilder sb = new StringBuilder();
Random random = new Random();
for (int i = 0; i < 10; i++) {
sb.append(random.nextInt(10)); // Append random digit to the telephone number
}
sb.append("-");
for (int i = 0; i < 3; i++) {
sb.append(random.nextInt(10)); // Append random digit to the telephone number
}
return sb.toString();
}
}
生成的文件在该位置上：
二.bulk 批量导入数据
public class EsConnectionExample {
public static void main(String[] args) throws IOException {
// 创建客户端
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("ip", port, "http"))); // 修改为你的ES地址和端口
//bulk 导入
//这块改成你的文件的地址
try (BufferedReader reader = new BufferedReader(new FileReader("user.json"))) {
String line;
// 构造 BulkRequest 对象并添加要导入的文档
BulkRequest request = new BulkRequest();
while ((line = reader.readLine()) != null) {
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("name", line)
.field("age", line)
.field("sex" , line)
.field("telephone", line)
.endObject();
//这块改成你的索引名字
IndexRequest indexRequest = new IndexRequest("my_index")
.source(builder);
request.add(indexRequest);
}
// 发送 BulkRequest 请求
BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);
if (response.hasFailures()) {
System.out.println("Failed to import documents.");
} else {
System.out.println("Documents imported successfully!");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭 ElasticSearch 客户端连接
client.close();
}
}
}
此时已经插入了
