org.elasticsearch.client
elasticsearch-rest-high-level-client
7.0.0
org.elasticsearch
elasticsearch
7.0.0
org.elasticsearch.client
transport
7.0.0
org.elasticsearch.plugin
transport-netty4-client
7.0.0
import org.elasticsearch.action.search.SearchResponse;
import java.util.List;
import java.util.Map;
/**
* @Author: gh
* @Description: elasticsearch查询数据(分页)、添加数据、修改数据、删除数据、展示所有库表、统计每张表的记录总条数;
* es的使用率,已使用的存储空间大小。按任意字段进行模糊查询。获取所有字段的名称。
*/
public interface EsDao {
/**
* 分页查询
* @param index 索引名称
* @param pageSize 页的大小
* @param pageNum 第几页
* @return
*/
public SearchResponse getAllRowsBySearchAfter(String index, Integer pageSize, Integer pageNum);
public SearchResponse getAllRowsByFromSize(String index, Integer pageSize, Integer pageNum);
public List> getAllRowsByScroll(String index);
/**
* 统计每个index下的document总数。
* count API的命令：GET /index_name/type_name/_count
* @param indexes 查询一个或多个index
* @return
*/
public long totalCount(String...indexes);
/**
* 查询某条document是否存在。
* @param index 索引名称
* @param id _id字段对应的主键值。
* @return true存在，false不存在。
*/
public boolean docIsExist(String index,String id);
/**
* 新增一条document。
* @param index 索引名称
* @param kvs 各个字段名称（key）和对应的值（value）
* @return 200/201表示新增成功；其他如400表示新增失败。
*/
public int insertDoc(String index,Map kvs);
/**
* 删除一条document。
* @param index 索引名称
* @param id _id字段对应的主键值。
* @return 200/201表示删除成功；其他如400表示删除失败。
*/
public int deleteDoc(String index,String id);
/**
* 更新一条document
* @param index 索引名称
* @param id _id字段对应的主键值。
* @param kvs 各个字段名称（key）和对应的值（value）
* @return 200/201表示更新成功；其他如400表示更新失败。
*/
public int updateDoc(String index,String id,Map kvs);
/**
* es使用率 = cluster store(size_in_bytes) / (所有节点的磁盘可用空间 + size_in_bytes)
* size_in_bytes: "_all":{"total":{"store":{"size_in_bytes": ***}}}
* 某个节点的磁盘可用空间：（节点状态）"fs":{"total":{"available_in_bytes":***}}
* @return
*/
public double usedRate();
/**
* 获取某个cluster下所有的index名称列表。
* 命令：_cluster/state/routing_table
* @param clusterName 集群名称
* @return
*/
public Map> getClusterIndexes(String clusterName);
/**
* 获取某个index下所有的type名称列表。
* _type字段从elasticsearch 6.0.0版本开始过时，7.0版本后不再使用。
* @return
*/
public Map> getIndexTypes(String clusterName);
/**
* @return 全部已使用存储空间的大小
*/
public double storeSizeOfMB();
/**
* @param index 需要查询的索引名称
* @return （某个索引下）已使用存储空间的大小
*/
public double storeSizeOfMB(String index);
/**
* 暂行：1.统计某个集群（相当于数据库）下所有index（相当于表）的数量。默认集群时elasticsearch。
* 搁置：2.统计某个index（相当于数据库）下所有type（相当于表）的数量。
* @return 表的数量
*/
public int countTables();
/**
* 获取某个集群（相当于数据库）下所有的indice（相当于表）的名称列表。
* @return
*/
public List getTablenamesOfDB();
/**
* 按任意字段进行模糊查询
* @param indexName 索引名
* @param fieldName 字段名
* @param fieldValue 字段模糊值
*/
public SearchResponse queryByRandomField(String indexName,String fieldName,String fieldValue,
int pageSize,int pageNum);
/**
* @param indexName 索引名
* @param fieldName 字段名
* @param fieldValue 字段模糊值
* @return 模糊查询时返回的记录总数
*/
public long totalCountOfFuzzyQuery(String indexName,String fieldName,String fieldValue);
/**
* 获取 某个索引的所有字段名
* @param indexName 索引的名称。例如：table1
* @return map k:字段名，v:空字符串
*/
public Map getColumnNames(String indexName);
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.genius.pojo.pg.dto.DataBaseDTO;
import com.genius.util.StringUtil;
import com.genius.util.common.FileSizeUtil;
import com.genius.util.common.SizeUnitEnum;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.*;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.*;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.locationtech.jts.geom.Coordinate;
import org.springframework.util.StringUtils;
import java.io.*;
import java.sql.*;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Description: 使用es JDBC 方式进行 查询数据(分页)、添加数据、修改数据、删除数据、展示所有库表。
* 【注意】
* 1.Elasticsearch JDBC方式不提供连接池机制；可以使用第三方连接池的机制。
* 2.使用Elasticsearch x-pack-sql-jdbc的包会报错：
* java.sql.SQLInvalidAuthorizationSpecException:
* current license is non-compliant for [jdbc]
* 使用JDBC客户端，elasticsearch需要升级到白金级：https://www.elastic.co/cn/subscriptions
*/
public class EsDaoImpl implements EsDao{
RestHighLevelClient restClient = null;
DataBaseDTO dataBaseDTO = null;
public EsDaoImpl(DataBaseDTO dbd) {
try {
dataBaseDTO = dbd;
this.restClient = connectToES();
//initClient();
} catch (Exception e) {
e.printStackTrace();
}
}
public RestHighLevelClient getRestClient() {
return restClient;
}
public void setRestClient(RestHighLevelClient restClient) {
this.restClient = restClient;
}
/**
* 关闭连接
*/
public void close(){
try{
if(this.restClient != null){
restClient.close();
}
}catch(IOException e){
e.printStackTrace();
}
}
public boolean connected(){
try {
if(getRestClient() != null && getRestClient().ping(RequestOptions.DEFAULT)){
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
/**
* JDBC连接elasticsearch。http port=9200
* jdbc:es://[[http|https]://][host[:port]]/[prefix]<[?[option=value]&]
* e.g. jdbc:es://http://server:3456/?timezone=UTC&page.size=250
* 安全通信：SSL
* @return
* @throws SQLException
*/
/*private Connection connectToES_abandon() throws SQLException {
if(conn == null){
if(dataBaseDTO != null){
String url = "jdbc:es://"+dataBaseDTO.getIp()+":9200";
System.out.println(url);
EsDataSource ds = new EsDataSource();
ds.setUrl(url);
Properties props = new Properties();
*//*props.put("user", "test_admin");
props.put("password", "x-pack-test-password");*//*
ds.setProperties(props);
return ds.getConnection();
//return DriverManager.getConnection(url, props);
}
}
return conn;
}*/
/**
* 连接 elasticsearch 并获取 index信息
* @return
* @throws SQLException
*/
private RestHighLevelClient connectToES() {
try {
String esIP = dataBaseDTO.getIp();
//http port = 9200
String esPort = dataBaseDTO.getHost();
//索引。
//String index = dataBaseDTO.getDbName();
//TODO: 用户名、密码做校验。目前cluster是默认的。
return new RestHighLevelClient(
RestClient.builder(new HttpHost(esIP, Integer.valueOf(esPort), "http")));
//flag = indexIsExist(index);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
@Override
public SearchResponse getAllRowsByScrollScan(String index, Integer pageSize, Integer pageNum) {
return getAllRowsByScrollScan(null,index,pageSize,pageNum);
}
private SearchResponse getAllRowsByScrollScan(SearchSourceBuilder sourceBuilder,String index,
Integer pageSize,Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
SearchRequest search = new SearchRequest(index);
SearchResponse resp = null;
if(sourceBuilder == null){
sourceBuilder = new SearchSourceBuilder();
}
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder)
.searchType(SearchType.QUERY_THEN_FETCH)
.scroll(TimeValue.timeValueSeconds(60));
try {
resp = restClient.search(search, RequestOptions.DEFAULT);
SearchHit[] hits1 = resp.getHits().getHits();
//查询结果太少，直接返回
if(hits1 == null || hits1.length < pageSize){
return resp;
}
if(pageNum > 1){
String scrollId = resp.getScrollId();
for(int i=1;i> getAllRowsByScroll(String indexName, String column, String value) {
List> collect = new ArrayList<>();
final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));
SearchResponse resp = null;
SearchRequest search = new SearchRequest(indexName);
search.scroll(scroll);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
try {
setQuerySearchBuilder(sourceBuilder,indexName,column,value);
sourceBuilder.size(100);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder)
.searchType(SearchType.QUERY_THEN_FETCH)
.scroll(TimeValue.timeValueSeconds(60));
resp = restClient.search(search, RequestOptions.DEFAULT);
assert resp != null;
String scrollId;
int count = 0;
do {
count++;
System.out.println("=================="+count);
collect.addAll(Arrays.stream(resp.getHits().getHits()).map(m->{
Map oneRowData = m.getSourceAsMap(); //sourceAsMap 可能为null
if(oneRowData != null){
oneRowData.put("_id", m.getId());
//oneRowData.put("_type", hit.getType());
}
return oneRowData;
}).collect(Collectors.toList()));
scrollId = resp.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
resp = restClient.scroll(scrollRequest, RequestOptions.DEFAULT);
} while (resp.getHits().getHits().length != 0);
//清除滚屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以选择setScrollIds()将多个scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
restClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
if (collect.size() == 0 || collect == null) {
return null;
}
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
return collect;
}
@Override
public List> getAllRowsByScroll(String index) {
List> collect = new ArrayList<>();
final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
search.scroll(scroll);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.size(100);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
assert resp != null;
String scrollId;
int count = 0;
do {
Arrays.stream(resp.getHits().getHits()).forEach(hit->{
Map map=hit.getSourceAsMap();
map.put("_id",hit.getId());
collect.add(map);
});
scrollId = resp.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
resp = restClient.scroll(scrollRequest, RequestOptions.DEFAULT);
} while (resp.getHits().getHits().length != 0);
//清除滚屏
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
//也可以选择setScrollIds()将多个scrollId一起使用
clearScrollRequest.addScrollId(scrollId);
restClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
if (collect.size() == 0 || collect == null) {
return null;
}
} catch (IOException e) {
e.printStackTrace();
}
return collect;
}
@Override
public SearchResponse getAllRows(String index) {
//如果不判断索引是否存在，可能会报错：IndexNotFoundException: no such index
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return resp;
}
@Override
public SearchResponse getAllRowsByFromSize(String index, Integer pageSize, Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
//如果不判断索引是否存在，可能会报错：IndexNotFoundException: no such index
SearchResponse resp = null;
SearchRequest search = new SearchRequest(index);
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分页查询
sourceBuilder.from(pageSize*pageNum-pageSize);
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
//当使用 TransportClient 时的API.
/*if(indexIsExist(index)) {
resp = client.prepareSearch(index)
.setQuery(QueryBuilders.matchAllQuery())
.setFrom(pageSize*pageNum-pageSize).setSize(pageSize)
//如果不排序，每次返回的结果是乱序的
.addSort("_id", SortOrder.ASC)
//.setPostFilter(QueryBuilders.rangeQuery("doc.offset").from(7000).to(10000))
.get();
}*/
return resp;
}
@Override
public SearchResponse getAllRowsBySearchAfter(String index, Integer pageSize, Integer pageNum) {
if(pageSize==null || pageSize<1){
pageSize = 10;
}
if(pageNum==null || pageNum<1){
pageNum = 1;
}
//============NOTE: API方式一---使用high level API
SearchResponse resp = null;
Object[] sortValues = null;
int counter = 0;
try {
//TODO:问题-pageNum大时，速度非常慢！
do{
//计数：当前是第几页
counter += 1;
SearchRequest search = new SearchRequest(index);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分页查询
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
//第一次搜索时，没有search_after参数。
if(sortValues != null){
sourceBuilder.searchAfter(sortValues);
}
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);
SearchHits hits = resp.getHits();
//当搜索没有结果时，hitSize = 0
int hitSize= hits.getHits().length;
if(hitSize == 0){
break;
}
SearchHit hit = hits.getHits()[hitSize - 1];
sortValues = hit.getSortValues();
}while(counter < pageNum);
} catch (IOException e) {
e.printStackTrace();
}
return resp;
//============NOTE: API方式二---使用low level API
/*Object tiebreaker = null; //查询的最后一个结果的ID
String jsonParam = null;
try {
do{
if(tiebreaker == null){
jsonParam = "{\"size\": "+pageSize+
",\"sort\": [{\"_id\": \"desc\"}]}";
}else{
jsonParam = "{\"size\": "+pageSize+"," +
"\"search_after\":"+tiebreaker+","+
"\"sort\": [{\"_id\": \"desc\"}]}";
}
//search_after请求
Request req = new Request("get", index+"/_search");
req.setJsonEntity(jsonParam);
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
JSONArray jsonArray = jo.getJSONObject("hits").getJSONArray("hits");
int dataSize = jsonArray.size();
if(dataSize > 0){
*//* XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, jsonArray.toJSONString());*//*
//返回的分页结果集
Object lastResult = jsonArray.get(jsonArray.size() - 1);
if(lastResult instanceof JSONObject){
tiebreaker = ((JSONObject) lastResult).getJSONArray("sort");
}
}else{
break;
}
}else{
break;
}
}while(true);
} catch (IOException e) {
e.printStackTrace();
}
return null;*/
}
public Map extractResponse(SearchResponse resp){
Map rowDatas = new HashMap<>();
List> data = new ArrayList<>();
Set fields = new HashSet<>();
if(resp != null){
SearchHits hits = resp.getHits();
Iterator ite = hits.iterator();
while(ite.hasNext()){
SearchHit hit = ite.next();
//sourceAsMap 可能为null
Map oneRowData = hit.getSourceAsMap();
if(oneRowData != null){
oneRowData.put("_id", hit.getId());
//oneRowData.put("_type", hit.getType());
//[NOTE:]前端需要显示几何对象的经纬度数值字符串
//[NOTE:]有routing路由的，需要加入路由
}
fields.addAll(oneRowData.keySet());
data.add(oneRowData);
}
}
rowDatas.put("data", data);
rowDatas.put("fields", fields);
rowDatas.put("pk", "_id");
return rowDatas;
}
private long getMaxresult(String index){
//ClusterGetSettingsRequest cgsr = new ClusterGetSettingsRequest();
long maxResult = 10000;
try {
Request req = new Request("get", index+"/_settings");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//查询设置
JSONObject settingObj = jo.getJSONObject(index)
.getJSONObject("settings")
.getJSONObject("index");
String value = settingObj.getString("max_result_window");
if(value == null){
return maxResult; //默认大小10000
}
maxResult = Long.valueOf(value);
}
} catch (IOException e) {
e.printStackTrace();
}
return maxResult;
}
/**
* 判断索引是否存在
* @param index 索引名称
* @return boolean
*/
private boolean isIndexExist(String index){
try{
if(!StringUtils.isEmpty(index)){
GetIndexRequest gir = new GetIndexRequest(index);
return restClient.indices().exists(gir, RequestOptions.DEFAULT);
}
}catch(Exception e){
e.printStackTrace();
}
//当使用 TransportClient 时的API.
//BasicConfigurator.configure();
/*AdminClient admin = client.admin();
IndicesAdminClient indices = admin.indices();
IndicesExistsRequestBuilder ierb = indices.prepareExists(index);
ActionFuture future = ierb.execute();
IndicesExistsResponse resp = future.actionGet();
if(resp.isExists()){
System.out.format("index [%s] is exist...\n", index);
return true;
}else {
System.out.format("index [%s] is NOT exist...\n", index);
return false;
}*/
return false;
}
//插入数据时，需要注意数据类型是否匹配。
@Override
public int insertDoc(String index,Map jsonMap) {
try {
if(jsonMap != null){
//将几何类型的转为特定对象
Map columns = getColumnNames(index);
Set keys = jsonMap.keySet();
for (String key : keys) {
EsFieldType eft = EsFieldType.GEO_POINT;
if(eft.getType().equals(columns.get(key))){
//如果是几何类型，就转换
Object transferedField = eft.getTransferedField(jsonMap.get(key).toString());
if(transferedField==null){
return 0;
}
jsonMap.put(key, transferedField);
}
}
//不设置id时，id会自动生成。
IndexRequest indexRequest = new IndexRequest(index).source(jsonMap);
//opType must be 'create' or 'index'.
// optype=index时，如果某ID对应的document已经存在，它将被替换。
//NOTE: 要么需要判断该id是否存在，已存在则不新增；要么让id自动生成。
indexRequest.opType(DocWriteRequest.OpType.INDEX);
//indexRequest.timeout(TimeValue.timeValueSeconds(1));
//强制刷新，消除延迟
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
/*
*几何类型的字段，可能报错：需要按几何对象格式去存储。
*Elasticsearch exception [type=mapper_parsing_exception,
* reason=failed to parse field [location] of type [geo_point]]]
*/
IndexResponse indexResponse = restClient.index(indexRequest, RequestOptions.DEFAULT);
int result = indexResponse.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //新增成功
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
@Override
public int deleteDoc(String index, String id) {
try {
DeleteRequest delRequest = new DeleteRequest(index,id);
// 如果存在routing,需要指定routing
String routing = getRouting(index, id);
if(routing != null){
delRequest.routing(routing);
}
//强制刷新，消除延迟
delRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
DeleteResponse delResp = restClient.delete(delRequest, RequestOptions.DEFAULT);
int result = delResp.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //删除成功
}
} catch (IOException e) {
e.printStackTrace();
}
return 0;//删除失败
}
@Override
public int updateDoc(String index, String id, Map kvs) {
try {
//将几何类型的转为特定对象
Map columns = getColumnNames(index);
Set keys = kvs.keySet();
for (String key : keys) {
EsFieldType eft = EsFieldType.GEO_POINT;
if(eft.getType().equals(columns.get(key))){
//如果是几何类型，就转换
Object transferedField = eft.getTransferedField(kvs.get(key).toString());
if(transferedField==null){
return 0;
}
kvs.put(key, transferedField);
}
}
UpdateRequest updateRequest = new UpdateRequest(index, id);
/*
* 报错：ElasticsearchStatusException[Elasticsearch exception [type=document_missing_exception
* 如果存在routing,需要指定routing
*/
String routing = getRouting(index, id);
if(routing != null){
updateRequest.routing(routing);
}
updateRequest.doc(kvs);
//NOTE:查询document，如果不存在，就不更新。
UpdateResponse updateResp = restClient.update(updateRequest, RequestOptions.DEFAULT);
//强制刷新，消除延迟
updateResp.setForcedRefresh(true);
int result = updateResp.status().getStatus();
if(result==RestStatus.OK.getStatus() || result == RestStatus.CREATED.getStatus()){
return 1; //更新成功
}
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
@Override
public boolean docIsExist(String index, String id) {
boolean flag = false;
try {
GetRequest getRequest = new GetRequest(index, id);
GetResponse getResp = restClient.get(getRequest, RequestOptions.DEFAULT);
flag = getResp.isExists();
} catch (IOException e) {
e.printStackTrace();
}finally {
return flag;
}
}
@Override
public long totalCount(String...indexes) {
long countNum = 0;
try {
//直接统计总数，没有设置search条件。
CountRequest countRequest = new CountRequest(indexes);
CountResponse countResp = restClient.count(countRequest, RequestOptions.DEFAULT);
countNum = countResp.getCount();
} catch (IOException e) {
e.printStackTrace();
}finally {
return countNum;
}
}
@Override
public double usedRate() {
//es使用率 = cluster store(size_in_bytes) / (所有节点的磁盘可用空间 + size_in_bytes)
double rate = 0.0;
try {
Request req = new Request("get", "_cluster/stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//查询整个集群的已用的存储空间大小（包括所有的index）
long totalIndexSizes = jo.getJSONObject("indices")
.getJSONObject("store")
.getLongValue("size_in_bytes");
//查询各个节点上的磁盘可用空间总和
long totalAvailableFSSizes = jo.getJSONObject("nodes")
.getJSONObject("fs")
.getLongValue("available_in_bytes");
System.out.println(totalIndexSizes+"==============="+totalAvailableFSSizes);
rate = (double)totalIndexSizes / (totalAvailableFSSizes + totalIndexSizes);
}
} catch (IOException e) {
e.printStackTrace();
}finally {
return Double.parseDouble(String.format("%.2f",rate*100));
}
}
@Override
public double storeSizeOfDB(SizeUnitEnum unit) {
return storeSizeOfDB(null,unit);
}
@Override
public double storeSizeOfDB(String index,SizeUnitEnum unit) {
try {
Request req = new Request("get", "_stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
if(StringUtils.isEmpty(index)){
//查询整个集群的已用的存储空间大小（包括所有的index）
long bytes = jo.getJSONObject("_all")
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
return FileSizeUtil.valueOf((double)bytes, unit);
}else{
//查询某个索引下已用的存储空间大小
if(isIndexExist(index)){
long bytes = jo.getJSONObject("indices")
.getJSONObject(index)
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
return FileSizeUtil.valueOf((double)bytes, unit);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0.0;
}
@Override
public double storeSizeOfTbl(String[] indices, SizeUnitEnum unit) {
try {
if(indices != null ){
Request req = new Request("get", "_stats");
RestClient client = restClient.getLowLevelClient();
Response resp = client.performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
JSONObject indiceJO = jo.getJSONObject("indices");
//查询某个索引下已用的存储空间大小
long bytes = 0L;
for (String index : indices) {
//判断索引是否存在
//if(isIndexExist(index)){ }
if(indiceJO.get(index) != null){
bytes += indiceJO
.getJSONObject(index)
.getJSONObject("total")
.getJSONObject("store")
.getLongValue("size_in_bytes");
}
}
return FileSizeUtil.valueOf((double)bytes, unit);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return 0.0;
}
@Override
public int countTables() {
//默认的集群：elasticsearch
return countIndices("elasticsearch");
}
private int countIndices(String clusterName){
//ClusterStatsRequest req = new ClusterStatsRequest();
//Request req = new Request("get", "_cluster/stats");
return getTablenamesOfDB().size();
}
@Override
public List getTablenamesOfDB() {
List nameList = new ArrayList<>();
List filteredNameList = new ArrayList<>();
try {
Request req = new Request("get", "_stats");
Response resp = restClient.getLowLevelClient().performRequest(req);
HttpEntity entity = resp.getEntity();
if(entity != null){
InputStream content = entity.getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(content));
String line = null;
StringBuffer sb = new StringBuffer();
while((line = reader.readLine())!=null){
sb.append(line);
}
JSONObject jo = JSON.parseObject(sb.toString());
//获取所有的indices
JSONObject indices = jo.getJSONObject("indices");
nameList.addAll(indices.keySet());
}
//过滤掉自动生成的index
for (String idx : nameList) {
if(!idx.startsWith(".")){
filteredNameList.add(idx);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return filteredNameList;
}
@Override
public SearchResponse queryByRandomField(String indexName, String fieldName, String fieldValue,
int pageSize,int pageNum) {
SearchResponse resp = null;
try {
/* SearchRequest search = new SearchRequest(indexName);
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分页查询
sourceBuilder.from(pageSize*pageNum-pageSize);
sourceBuilder.size(pageSize);
sourceBuilder.sort("_id", SortOrder.ASC);
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
search.source(sourceBuilder);
resp = restClient.search(search, RequestOptions.DEFAULT);*/
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
resp = getAllRowsByScrollScan(sourceBuilder, indexName, pageSize, pageNum);
}catch(Exception e){
e.printStackTrace();
}
return resp;
}
private void setQuerySearchBuilder(SearchSourceBuilder sourceBuilder,
String indexName,
String fieldName, String fieldValue) throws Exception{
String[]numeric = {"long","integer","short","byte","double","float","half_float","scaled_float"};
List numericTypes = Arrays.asList(numeric);
//获取字段类型
String fieldType = getFieldType(indexName, fieldName);
if("text".equals(fieldType) || "keyword".equals(fieldType)){
//以*开头的wildcardQuery非常慢，不建议使用
//模糊查询：fuzzy，wildcard: 只针对text，keyword类型字段。
//text类型：可以使用fuzzy模糊查询。keyword类型，使用fuzzy查询失效。
sourceBuilder.query(QueryBuilders.wildcardQuery(fieldName,"*" + fieldValue + "*"));
//sourceBuilder.query(QueryBuilders.fuzzyQuery(fieldName, fieldValue).fuzziness(Fuzziness.AUTO));
//sourceBuilder.query(QueryBuilders.matchQuery(fieldName, fieldValue).fuzziness(Fuzziness.AUTO));
}else if(numericTypes.contains(fieldType)){
if(StringUtil.isNumeric(fieldValue)){
sourceBuilder.query(QueryBuilders.rangeQuery(fieldName).gte(fieldValue));
}
}else if("geo_point".equals(fieldType)){
//Geo fields do not support exact searching, use dedicated geo queries instead
if(fieldValue != null){
String[] locations = fieldValue.split(",");
if(locations.length == 4){
double top = StringUtil.isNumeric(locations[0].trim())?Double.valueOf(locations[0].trim()):90;
double left = StringUtil.isNumeric(locations[1].trim())?Double.valueOf(locations[1].trim()): -180;
double bottom = StringUtil.isNumeric(locations[2].trim())?Double.valueOf(locations[2].trim()) :-90;
double right = StringUtil.isNumeric(locations[3].trim())?Double.valueOf(locations[3].trim()): 180;
sourceBuilder.query(QueryBuilders.geoBoundingBoxQuery(fieldName)
.setCorners(top, left, bottom, right));
}
}
}else if("geo_shape".equals(fieldType)){
//Geo fields do not support exact searching, use dedicated geo queries instead
if(fieldValue != null){
String[] locations = fieldValue.split(",");
if(locations.length == 4){
double top = StringUtil.isNumeric(locations[0].trim())?Double.valueOf(locations[0].trim()):90;
double left = StringUtil.isNumeric(locations[1].trim())?Double.valueOf(locations[1].trim()): -180;
double bottom = StringUtil.isNumeric(locations[2].trim())?Double.valueOf(locations[2].trim()) :-90;
double right = StringUtil.isNumeric(locations[3].trim())?Double.valueOf(locations[3].trim()): 180;
List coordinates = new CoordinatesBuilder().coordinate(left, top)
.coordinate(right, bottom).build();
GeometryCollectionBuilder gcb = new GeometryCollectionBuilder();
gcb.coordinates(coordinates);
sourceBuilder.query(QueryBuilders.geoWithinQuery(fieldName, gcb));
}
}
}else{
sourceBuilder.query(QueryBuilders.matchQuery(fieldName, fieldValue));
}
}
@Override
public long totalCountOfFuzzyQuery(String indexName, String fieldName, String fieldValue) {
long counter = 0;
try {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//模糊查询：fuzzy，wildcard
setQuerySearchBuilder(sourceBuilder,indexName,fieldName,fieldValue);
//直接统计总数，设置search条件。
CountRequest countRequest = new CountRequest(new String[]{indexName},sourceBuilder);
CountResponse countResponse = restClient.count(countRequest, RequestOptions.DEFAULT);
counter = countResponse.getCount();
} catch (Exception e) {
e.printStackTrace();
}
return counter;
}
//获取字段类型
private String getFieldType(String indice,String fieldName)throws IOException{
Map mappings = getMappingInfo(indice);
Map source = mappings.get(indice).getSourceAsMap();
Object properties = source.get("properties");
if(properties instanceof LinkedHashMap){
LinkedHashMap map = (LinkedHashMap)properties;
Object field = map.get(fieldName);
if(field instanceof LinkedHashMap){
LinkedHashMap fieldMap = (LinkedHashMap)field;
String type = fieldMap.get("type").toString();
return type;
}
}
return null;
}
/**
* 获取mapping信息
* @param indice
* @return
* @throws IOException
*/
public Map getMappingInfo(String indice) throws IOException{
GetMappingsRequest gmr = new GetMappingsRequest();
gmr.indices(indice);
GetMappingsResponse resp = restClient.indices()
.getMapping(gmr, RequestOptions.DEFAULT);
Map mappings = resp.mappings();
return mappings;
}
@Override
public Map getColumnNames(String indexName) {
Map columnNames = new HashMap<>();
GetMappingsRequest mappingsRequest = new GetMappingsRequest().indices(indexName);
try {
GetMappingsResponse mappingsResponse = restClient.indices()
.getMapping(mappingsRequest, RequestOptions.DEFAULT);
Map mappings = mappingsResponse.mappings();
if(mappings != null){
MappingMetaData metaData = mappings.get(indexName);
if(metaData != null){
Map sourceAsMap = metaData.getSourceAsMap();//properties
if(sourceAsMap != null){
Collection collection = sourceAsMap.values();//Object = map
Map tmp = new HashMap<>();
Iterator ite = collection.iterator();
while (ite.hasNext()){
tmp.putAll((Map)ite.next());
}
Set fields = tmp.keySet();
//提取字段名和类型
for (String field : fields) {
Map fieldMap = (Map)tmp.get(field);
columnNames.put(field, fieldMap.get("type"));
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return columnNames;
}
/**
* 根据索引ID查询对应document，并返回routing内容
* @param index
* @param id
*/
private String getRouting(String index, String id){
SearchResponse resp = queryByRandomField(index, "_id", id, 1, 1);
if(resp != null){
SearchHits hits = resp.getHits();
Iterator ite = hits.iterator();
while(ite.hasNext()){
SearchHit hit = ite.next();
DocumentField df = hit.field("_routing");
if(df != null){
List values = df.getValues();
if(values != null){
String valStr = values.toString();
//将routing转成字符串返回
return valStr.substring(1, valStr.length()-1);
}
}
}
}
return null;
}
@Override
public Map> getClusterIndexes(String clusterName) {
return null;
}
@Override
public Map> getIndexTypes(String clusterName) {
return null;
}
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.util.regex.Pattern;
/**
* @Description: elasticsearch字段类型：
* https://www.elastic.co/guide/en/elasticsearch/reference/6.0/mapping-types.html
* 该枚举类用于转换elasticsearch特殊字段的数据值
*/
public enum EsFieldType {
/**
* 几何点类型，接收经纬度数据对。
* geo_point有四种形式：
* #对象 "location": {"lat": 41.12,"lon": -71.34}
* #字符串 "location": "41.12,-71.34"
* #geohash "location": "drm3btev3e86"
* #数组 "location": [ -71.34, 41.12 ]
*/
GEO_POINT("geo_point"){
@Override
public Object fieldValueTransfer(String fieldValue) {
String rex1 = "\\{(\\s)*\"lat\"(\\s)*:(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,"+
"(\\s)*\"lon\"(\\s)*:(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*\\}";
String rex2 = "(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?";
String rex3 = "^[a-z0-9]+$";
String rex4 = "^\\[(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*,(\\s)*(\\-|\\+)?\\d+(\\.\\d+)?(\\s)*\\]$";
if(match(rex1,fieldValue)){
//json object
return parseJsonToGeopoint(fieldValue);
}else if(match(rex4,fieldValue)){
//array
return JSON.parseArray(fieldValue);
}else if(match(rex2,fieldValue)){
//string
return fieldValue;
}else if(match(rex3,fieldValue)){
//geohash,不能有大写的英文字母
return GeoPoint.fromGeohash(fieldValue);
}else{
return null;
}
}
private GeoPoint parseJsonToGeopoint(String jsonStr){
//{"lat": 41.12,"lon": -71.34}
JSONObject jo = JSON.parseObject(jsonStr);
//直接返回JSONObject也可以，但经纬度前后顺序会颠倒。
return new GeoPoint(jo.getDoubleValue("lat"), jo.getDoubleValue("lon"));
}
};
EsFieldType(String type) {
this.type = type;
}
public Object getTransferedField(String fieldValue){
return fieldValueTransfer(fieldValue);
}
public boolean match(String rex,String input){
Pattern p = Pattern.compile(rex);
return p.matcher(input).matches();
}
private String type;
public String getType() {
return type;
}
protected abstract Object fieldValueTransfer(String fieldValue);
}
