Elasticsearch结合Java使用之官方API方法
Elasticsearch结合Java使用之官方API方法
- 一、前期准备
- 二、新建索引
- 三、查询文档
- 四、新增文档
- 五、更新文档
- 六、删除文档
- 七、条件查询（布尔查询）
- 八、查询所有文档
- 九、聚合查询
- 十、批量导入数据
一、前期准备
- 新建一个Maven工程，推荐使用Springboot来搭建项目，在pom.xml中引入Elasticsearch依赖：
注：这里不用指定版本号，Springboot会根据自身的版本来自动加载相对应版本的Elasticsearch，你可以根据下面的关联关系图来选择Springboot的版本。<dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> //7.3.0 //如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-client</artifactId> //7.3.0 //如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency> <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> //7.3.0 //如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency>Elasticsearch Spring Boot 7.9.3 2.4.x 7.6.2 2.3.x 6.8.12 2.2.x 6.2.2 2.1.x 5.5.0 2.0.x 2.4.0 1.5.x - 在resources下面创建一个配置文件application.yml，里面配置好Elasticsearch的连接属性：
然后新建一个package，创建一个配置类去加载配置属性：elasticsearch: ip: 127.0.0.1 port: 9200 //如果你安装的Elasticsearch没有设置用户名和密码的话，以下两个属性则填任意值就可以 username: elastic password: 123456
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @Component @ConfigurationProperties(prefix = "elasticsearch") public class ElasticSearchConfig { private String ip; private int port; private String username; private String password; public String getIp() { return ip; } public void setIp(String ip) { this.ip = ip; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } } - 我们使用配置方式注入到Spring容器中，这样在后续使用过程中不需要频繁地手动创建和关闭连接。新建一个package，在下面创建如下两个类：
import com.common.ElasticSearchConfig; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; @Component public class ESClientSpringFactory { public final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); public static int CONNECT_TIMEOUT_MILLIS = 1000; public static int SOCKET_TIMEOUT_MILLIS = 30000; public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; public static int MAX_CONN_PER_ROUTE = 10; public static int MAX_CONN_TOTAL = 30; private static HttpHost[] HTTP_HOST; private RestClientBuilder builder; private RestClient restClient; private RestHighLevelClient restHighLevelClient; private static ESClientSpringFactory esClientSpringFactory = new ESClientSpringFactory(); @Autowired private ElasticSearchConfig elasticSearchConfig; private ESClientSpringFactory() {} public static ESClientSpringFactory build(HttpHost[] httpHost, Integer maxConnectNum, Integer maxConnectPerRoute) { HTTP_HOST = httpHost; MAX_CONN_TOTAL = maxConnectNum; MAX_CONN_PER_ROUTE = maxConnectPerRoute; return esClientSpringFactory; } public static ESClientSpringFactory build(HttpHost[] httpHost, Integer connectTimeOut, Integer socketTimeOut, Integer connectionRequestTime, Integer maxConnectNum, Integer maxConnectPerRoute) { HTTP_HOST = httpHost; CONNECT_TIMEOUT_MILLIS = connectTimeOut; SOCKET_TIMEOUT_MILLIS = socketTimeOut; CONNECTION_REQUEST_TIMEOUT_MILLIS = connectionRequestTime; MAX_CONN_TOTAL = maxConnectNum; MAX_CONN_PER_ROUTE = maxConnectPerRoute; return esClientSpringFactory; } public void init() { builder = RestClient.builder(HTTP_HOST); setConnectTimeOutConfig(); setMutiConnectConfig(); restClient = builder.build(); restHighLevelClient = new RestHighLevelClient(builder); } public void setConnectTimeOutConfig() { builder.setRequestConfigCallback(requestConfigBuilder -> { requestConfigBuilder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); requestConfigBuilder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS); requestConfigBuilder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS); return requestConfigBuilder; }); } public void setMutiConnectConfig() { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticSearchConfig.getUsername(), elasticSearchConfig.getPassword())); builder.setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.setMaxConnTotal(MAX_CONN_TOTAL); httpClientBuilder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE); httpClientBuilder.disableAuthCaching(); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); return httpClientBuilder; }); } public RestClient getClient() { return restClient; } public RestHighLevelClient getRhlClient() { return restHighLevelClient; } public void close() { if (restClient != null) { try { restClient.close(); } catch (IOException e) { e.printStackTrace(); } } } }import com.alibaba.fastjson.JSONArray; import com.common.ElasticSearchConfig; import org.apache.http.HttpHost; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class ESConfig { private static final String HTTP_SCHEME = "http"; @Autowired private ElasticSearchConfig elasticSearchConfig; @Bean public HttpHost[] httpHost() { JSONArray array = new JSONArray(); if (elasticSearchConfig.getIp().indexOf(",") != -1) { for (String ip : elasticSearchConfig.getIp().split(",")) { array.add(new HttpHost(ip, elasticSearchConfig.getPort(), HTTP_SCHEME)); } } else { array.add(new HttpHost(elasticSearchConfig.getIp(), elasticSearchConfig.getPort(), HTTP_SCHEME)); } HttpHost[] hosts = array.toArray(new HttpHost[0]); return hosts; } @Bean(initMethod = "init", destroyMethod = "close") public ESClientSpringFactory getFactory() { return ESClientSpringFactory.build(httpHost(), 300, 100); } @Bean(name = "highLevelClient") public RestHighLevelClient highLevelClient() { return getFactory().getRhlClient(); } }
二、新建索引
public class CreateIndexes {
@Autowired
private RestHighLevelClient highLevelClient;
public void createDemoIndex(String indexname) {
CreateIndexRequest request = new CreateIndexRequest(indexname);
try {
//判断是否已存在该索引，如果存在则先删除
GetIndexRequest exist = new GetIndexRequest(indexname);
if (highLevelClient.indices().exists(exist, RequestOptions.DEFAULT)) {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexname);
highLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
}
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("demoName")
.field("type", "text")
.field("analyzer", "keyword")
.endObject()
.startObject("demoValue")
.field("type", "text")
.field("analyzer", "keyword")
.endObject()
.startObject("demoNumber")
.field("type", "integer")
.endObject()
.startObject("demoTime")
.field("type", "date")
.endObject()
.endObject()
.endObject();
request.mapping(builder);
highLevelClient.indices().create(request, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
三、查询文档
public class GetDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public DemoBO getDocument(String indexName) {
DemoBO bo = new DemoBO();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//searchSourceBuilder.query(QueryBuilders.matchAllQuery()).from(0).size(10).sort("demoTime", SortOrder.DESC);//查询全部文档用此方法
searchSourceBuilder.query(QueryBuilders.matchPhraseQuery(indexName, "abc")).from(0).size(10).sort("demoTime", SortOrder.DESC);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits searchHits = searchResponse.getHits();
if (searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return bo;
}
}
四、新增文档
public class PutDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void putDocument(String indexName, String id) {
try {
IndexRequest indexRequest = new IndexRequest(indexName);
//IndexRequest indexRequest = new IndexRequest(indexName).id(id);//如果你要自己定义文档ID的值，可以这么写，否则Elasticsearch会自动帮你生成一个，不过官方也是不建议让你来建。
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//此句功能为立即刷新索引文档，不然你会在页面上发现我明明已经新增一个文档了，却要过个几秒钟再刷新页面才会显示出来。
Map<String, Object> map = new HashMap<>();
map.put("demoName", "abc");
map.put("demoValue", "123");
map.put("demoNumber", 1);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
map.put("demoTime", sdf.format(new Date()));
indexRequest.source(map);
highLevelClient.index(indexRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
五、更新文档
public class PostDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void postDocument(String indexName, String id) {
try {
UpdateRequest updateRequest = new UpdateRequest(indexName, id);
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//跟上面新增文档那里一样的作用
Map<String, Object> map = new HashMap<>();
map.put("demoName", "def");
map.put("demoValue", "456");
map.put("demoNumber", 2);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
map.put("demoTime", sdf.format(new Date()));
updateRequest.doc(map);
highLevelClient.update(updateRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
六、删除文档
public class DeleteDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void deleteDocument(String indexName, String id) {
try {
DeleteRequest deleteRequest = new DeleteRequest(indexName, id);
deleteRequest .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//跟上面新增文档那里一样的作用
highLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
七、条件查询（布尔查询）
public class BoolGetDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public DemoBO boolGetDocument(String indexName) {
DemoBO bo = new DemoBO();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("demoName", "abc"));
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery("demoValue", "123"));
boolQueryBuilder.should(QueryBuilders.rangeQuery("demoNumber").gte(1)).should(QueryBuilders.rangeQuery("demoNumber").lte(5));
searchSourceBuilder.query(boolQueryBuilder).from(0).size(10).sort("demoTime", SortOrder.DESC);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits searchHits = searchResponse.getHits();
if (searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return bo;
}
}
八、查询所有文档
public class GetAllDocuments {
@Autowired
private RestHighLevelClient highLevelClient;
public List<DemoBO> getAllDocuments(String indexName) {
List<DemoBO> list = new ArrayList<>();
try {
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery()).size(1000);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
StringBuilder scrollId = new StringBuilder(searchResponse.getScrollId());
SearchHits searchHits = searchResponse.getHits();
while (searchHits.getHits() != null && searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
DemoBO bo = new DemoBO();
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
list.add(bo);
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId.toString());
scrollRequest.scroll(scroll);
searchResponse = highLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId.replace(0, scrollId.length(), searchResponse.getScrollId());
searchHits = searchResponse.getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId.toString());
highLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
}
九、聚合查询
public class GetAggregation {
@Autowired
private RestHighLevelClient highLevelClient;
public List<DemoBO> getAggregation (String indexName) {
List<DemoBO> list = new ArrayList<>();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
AggregationBuilder aggregation = AggregationBuilders.terms("qc").field("demoName.keyword").size(10).subAggregation(AggregationBuilders.topHits("tj").sort("demoName.keyword", SortOrder.DESC).size(1));//"qc"和"tj"是自定义名称，可以自己取，"qc"那部分指的是去重，"tj"那部分指的是统计，相当于SQL查询中的Group by。
searchSourceBuilder.query(QueryBuilders.matchPhraseQuery("demoValue", "123")).aggregation(aggregation).size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
Terms terms = searchResponse.getAggregations().get("qc");
for (Bucket b: terms.getBuckets()) {
DemoBO bo = new DemoBO ();
bo.setDemoName(b.getKeyAsString());
TopHits top = b.getAggregations().get("tj");
for (SearchHit hit: top.getHits()) {
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue").toString());
} else {
bo.setDemoValue("");
}
}
list.add(bo);
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
}
十、批量导入数据
这边使用的是从Oracle数据库导入到Elasticsearch，可作为一个参考，实现方式是一样的。
public class OracleToES {
@Autowired
private RestHighLevelClient highLevelClient;
public void writeOracleDataToES(String indexName, String sql) {
BulkProcessor bulkProcessor = getBulkProcessor();
Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
try {
System.out.println("开始导入数据: " + indexName);
Class.forName("oracle.jdbc.driver.OracleDriver");
conn = DriverManager.getConnection("jdbc:oracle:thin:@127.0.0.1:1521:sjk", "demo", "123456");
ps = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
ps.setFetchSize(0);
rs = ps.executeQuery();
ResultSetMetaData colData = rs.getMetaData();
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> map;
int count = 0;
StringBuilder c = new StringBuilder("0");
Object v;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
while (rs.next()) {
count++;
map = new HashMap<>(100);
for (int i = 1; i <= colData.getColumnCount(); i++) {
c.replace(0, c.length(), colData.getColumnName(i));
if ("DEMO_TIME".equals(c.toString())) {//Oracle字段为Date类型时需要单独处理，否则导入会报错
if (rs.getDate(c.toString()) != null) {
v = sdf.format(rs.getDate(c.toString()));
} else {
v = null;
}
} else {
if (rs.getObject(c.toString()) != null) {
v = rs.getObject(c.toString());
} else {
v = null;
}
}
map.put(c.toString().toLowerCase(), v);//Oracle字段默认为大写，需要转换为小写
}
dataList.add(map);
if (count % 100000 == 0) {//每10万条数据提交一次
System.out.println("已提交数据量: " + count);
for (HashMap<String, Object> hashMap: dataList) {
bulkProcessor.add(new IndexRequest(indexName.toLowerCase()).source(hashMap));
}
map.clear();
dataList.clear();
}
}
for (HashMap<String, Object> hashMap: dataList) {//提交剩余的数据
bulkProcessor.add(new IndexRequest(indexName.toLowerCase()).source(hashMap));
}
System.out.println("成功提交数据量: " + count);
bulkProcessor.flush();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
rs.close();
ps.close();
conn.close();
boolean terminatedFlag = bulkProcessor.awaitClose(150L, TimeUnit.SECONDS);
System.out.println(terminatedFlag);
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
private BulkProcessor getBulkProcessor() {
BulkProcessor bulkProcessor = null;
try {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {
System.out.println("尝试导入数据量: " + request.numberOfActions());
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
System.out.println("成功导入数据量: " + request.numberOfActions() + " , id: " + executionId);
}
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
System.out.println("导入失败: " + failure + ", executionId: " + executionId);
}
};
BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer = (request, bulkListener) -> highLevelClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener);
builder.setBulkActions(1000);
builder.setBulkSize(new ByteSizeValue(100L, ByteSizeUnit.MB));
builder.setConcurrentRequests(10);
builder.setFlushInterval(TimeValue.timeValueSeconds(5L));
builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3));
bulkProcessor = builder.build();
} catch (Exception e) {
e.printStackTrace();
try {
bulkProcessor.awaitClose(100L, TimeUnit.SECONDS);
} catch (Exception e1) {
System.out.println(e1.getMessage());
}
}
return bulkProcessor;
}
}