elasticsearch使用restclient接口插入数据

elasticsearch使用restclient接口插入数据

首先定义要插入的数据。

1 定义一个student类，并json序列化，对于复杂的数据结构，使用以下的json格式化很方便的能得到我们需要的数据结构，

可以参考 http://blog.csdn.net/wslyk606/article/details/78325474 这篇文章。

package es;

import com.fasterxml.jackson.databind.annotation.JsonSerialize;

@JsonSerialize(using = StudentSerializer.class)
public class Student {
    private String name;
    private int age;
    private String addr;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public int getAge() {
        return age;
    }

    public void setAge(int age) {
        this.age = age;
    }


    public String getAddr() {
        return addr;
    }

    public void setAddr(String addr) {
        this.addr = addr;
    }

    @Override
    public String toString() {
        return "Student{" +
                "name='" + name + '\'' +
                ", age=" + age +
                ", addr='" + addr + '\'' +
                '}';
    }
}

2  json的具体实现。

package es;


import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;

import java.io.IOException;

public class StudentSerialize extends JsonSerializer {
    @Override
    public void serialize(Student student, JsonGenerator jGen, SerializerProvider serializerProvider) throws IOException {
        jGen.writeStartObject();
        jGen.writeStringField("name", student.getName());
        jGen.writeNumberField("age", student.getAge());
        jGen.writeStringField("address", student.getAddr());
        jGen.writeEndObject();
    }
}

3 构建es的restClient，用于进行插入数据。
ip地址和端口改成实际的url地址

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;


@Configuration
public class ESConfiguration {

    @Bean
    public RestClient restClient() {
        return RestClient.builder(HttpHost.create("http://192.168.10.33:9200"))
                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(60000)
                        .setSocketTimeout(2000))
                .build();
    }
}

4  插入代码实现
采用restClient的异步插入方法，实测插入效率比其他几种方式高。
但是对于大数据量会有数据丢失的情况，（可能是线程阻塞造成），采用了Thread.sleep(1);
每插入一次线程sleep 1毫秒，可以有效解决问题。

package es;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseListener;
import org.elasticsearch.client.RestClient;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.Collections;

@Repository
public class StudentInsertDao {

    @Autowired
    private RestClient restClient;

    private ObjectMapper objectMapper = new ObjectMapper();

    public void insert(Student student) {

//构建插入的index和type，index为students，用student的age作为type
//为了保证id的唯一，采用name+address+age来构造

String id = student.getName()+student.getAddr()+student.getAge();
        StringBuilder urlBuilder = new StringBuilder()
                .append("/")
                .append("students")
                .append("/")
                .append(student.getAge())
                .append("/")
                .append(id);

        HttpEntity entity;
        try {
            entity = new NStringEntity(objectMapper.writeValueAsString(student), ContentType.APPLICATION_JSON);
        } catch (JsonProcessingException e) {
            e.printStackTrace();
            return;
        }

        restClient.performRequestAsync("PUT",
                urlBuilder.toString(),
                Collections.emptyMap(),
                entity,
                new ResponseListener() {
                    @Override
                    public void onSuccess(Response response) {
                    }
                    @Override
                    public void onFailure(Exception exception) {
                        exception.printStackTrace();
                    }
                });
        try {
            Thread.sleep(1);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

5 测试代码

@RunWith(SpringJUnit4ClassRunner.class)
@WebAppConfiguration
@ContextConfiguration(locations = {"classpath:servlet-context.xml", "classpath:applicationContext.xml"})
public class StudentInsertDaoTest {    
    @Autowired
    private StudentInsertDao insertDao;
    
    @Test
    public void insert() throws Exception {
        
        Student student = new Student();
        student.setAge(12);
        student.setAddr("SH");
        student.setName("Jack");
        
        insertDao.insert(student);      
    }
}

插入之后可以查看es数据库中的数据。使用elasticsearch-head可以很方便的来查看索引信息和数据。

PS：此方式下大批量数据也会存在数据丢失的情况，可以使用Tcp方式进行批量入库，参考

http://blog.csdn.net/wslyk606/article/details/79413980