Elasticsearch通过restapi进行数据查询----java
做日志监控使用的是ELK一套的东西，然后将日志搜索和下载单独做了一个功能模块，所以就看着官网的API文档，写了查询的代码。
1.引入elasticsearch相关jar包，一定使用mvn来引入，如果自己去找很难找全的，因为他的包有43个。别问我为啥，因为自己去找包，然后搭建工程。花了挺多时间，一个mvn ，马上就下载好了。
2.去官网翻翻rest相关的接口，以及使用。
下面是一个搜索功能的代码，挺乱的，
先判断es中是否有index（索引），然后再进行搜索条件的拼接。
最后是用的GraphQl进行查询的。
/**
* 获取所有的index
* @return
* @throws IOException
* @throws SwordBaseCheckedException
*/
@Service(serviceName = "jkpt_LogSearchService_queryIndex")
public List> queryIndex(List> esinfolist) throws IOException, SwordBaseCheckedException{
//List result = new ArrayList();
/*HttpHost[] tmp1 = new HttpHost[1];
HttpHost tmp2 = new HttpHost("10.23.11.139", 9200,"http");
tmp1[0] = tmp2;
RestClient restClient = getLowClient(tmp1);*/
HttpHost[] hostarray = new HttpHost[esinfolist.size()];
int i = 0;
for(Map esinfo : esinfolist) {
HttpHost hosttmp = new HttpHost(esinfo.get("ipaddress"), Integer.parseInt(esinfo.get("esport")),"http");
hostarray[i] = hosttmp;
i++;
}
RestClient restClient = LogsearchUtil.getLowClient(hostarray);
Response response = restClient.performRequest("GET", "/_cat/indices?v&h=index",
Collections.singletonMap("pretty", "true"));
String[] allindexstr = EntityUtils.toString(response.getEntity()).split("\n");
List> allindexlist = new ArrayList>();
for(String indexstr : allindexstr) {
if("index".equals(indexstr) || indexstr.startsWith(".") || indexstr.startsWith("filebeat")) {
continue;
}
Map tmp = new HashMap();
tmp.put("code", indexstr);
tmp.put("caption", indexstr);
allindexlist.add(tmp);
}
//System.out.println("查询到的内容" + EntityUtils.toString(response.getEntity()));
return allindexlist;
}
/**
* 根据条件查询日志
* @param querMap
* @return
* @throws IOException
* @throws ParseException
* @throws SwordBaseCheckedException
*/
@Service(serviceName = "jkpt_LogSearchService_queryBySelect")
public Map queryBySelect(List> esinfolist,Map queryMap) throws ParseException, SwordBaseCheckedException, IOException{
logger.debug("=======================根据条件查询日志");
String ifshowAll = null;
if(queryMap.get("ifshowAll") != null) {
ifshowAll = (String) queryMap.get("ifshowAll");
queryMap.remove("ifshowAll");
}
String yhuuid =(String) queryMap.get("yhuuid");
queryMap.remove("yhuuid");
Map result = new HashMap();
SearchRequest searchRequest = new SearchRequest();
//每页显示的条数
int showsize = (Integer) queryMap.get("showsize");
queryMap.remove("showsize");
try {
List> indexlist = queryIndex(esinfolist);
if(indexlist == null || indexlist.size() == 0) {
logger.debug("======================当前es中无数据");
result.put("loglist", null);
result.put("totalHits", 0L);
result.put("pagesize", 0L);
return result;
}
//限制搜索的index
String[] indexnamearr = new String[indexlist.size()];
for(int i = 0 ;i tmp = indexlist.get(i);
String indexname = tmp.get("code");
indexnamearr[i] = indexname;
}
searchRequest = new SearchRequest(indexnamearr);
}catch(IOException ioex) {
result.put("error", "Connection refused");
return result;
}
/*MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("logLevel", "Error")
.fuzziness(Fuzziness.AUTO)
.prefixLength(3)
.maxExpansions(10);*/
//上面代码的另一种方式
/*MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("logLevel", "ERROR");
matchQueryBuilder.fuzziness(Fuzziness.AUTO);
matchQueryBuilder.prefixLength(3);
matchQueryBuilder.maxExpansions(10);
String[] indexs = {"logstash-2018.09.04","logstash-2018.08.27"};
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(matchQueryBuilder);*/
//获取es集群下的所有主机
HttpHost[] hostarray = new HttpHost[esinfolist.size()];
int i = 0;
for(Map esinfo : esinfolist) {
HttpHost hosttmp = new HttpHost(esinfo.get("ipaddress"), Integer.parseInt(esinfo.get("esport")),"http");
hostarray[i] = hosttmp;
i++;
}
RestHighLevelClient client = LogsearchUtil.getHighClient(hostarray);
//SearchRequest searchRequest = new SearchRequest();
//多index搜索 ----------暂时不适用index进行筛选
/*if(queryMap.get("indexsname") != null) {
String indexname = (String)queryMap.get("indexsname");
searchRequest = new SearchRequest(indexname);//index名称
queryMap.remove("indexsname");
}*/
/*MatchPhraseQueryBuilder mpq1 = QueryBuilders.matchPhraseQuery("logLevel",queryMap.get("logLevel"));
MatchPhraseQueryBuilder mpq2 = QueryBuilders.matchPhraseQuery("sessionId",queryMap.get("sessionId")); */
int nowpage = (Integer)queryMap.get("nowpage");
queryMap.remove("nowpage");
String sortway = (String) queryMap.get("sortway");
queryMap.remove("sortway");
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
//日志级别
if(queryMap.get("logLevel") != null) {
String logLevel = (String) queryMap.get("logLevel");
//yymc="{yyname="+yymc+"}";
//boolQueryBuilder.must(QueryBuilders.matchQuery("fields.yyname",yymc));
boolQueryBuilder.must(QueryBuilders.matchQuery("logLevel",logLevel ));
queryMap.remove("logLevel");
}
//应用名称
if(queryMap.get("yymc") != null) {
String yymc = (String) queryMap.get("yymc");
//yymc="{yyname="+yymc+"}";
boolQueryBuilder.must(QueryBuilders.matchQuery("fields.yyname",yymc));
//boolQueryBuilder.must(QueryBuilders.wildcardQuery("fields.yyname","*"+yymc+"*"));
queryMap.remove("yymc");
}
//关键字模糊查询处理
if(queryMap.get("keywordsearch") != null) {
String keywordsearch = (String) queryMap.get("keywordsearch");
boolQueryBuilder.must(QueryBuilders.matchQuery("message",keywordsearch).minimumShouldMatch("80%"));
//boolQueryBuilder.must(QueryBuilders.wildcardQuery("message","*"+keywordsearch+"*"));
queryMap.remove("keywordsearch");
}
/**
* 使用QueryBuilder
* termQuery("key", obj) 完全匹配
* termsQuery("key", obj1, obj2..) 一次匹配多个值
* matchQuery("key", Obj) 单个匹配, field不支持通配符, 前缀具高级特性
* multiMatchQuery("text", "field1", "field2"..); 匹配多个字段, field有通配符忒行
* matchAllQuery(); 匹配所有文件
*/
//查询资源整合对象
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
// 查询在时间区间范围内的结果 --------------处理时间区间
RangeQueryBuilder rangbuilder = QueryBuilders.rangeQuery("date.keyword");
boolean ifhasstar = false;
if(queryMap.get("starttime") != null) {
ifhasstar = true;
}
if(queryMap.get("endtime") != null) {
String endtime = (String) queryMap.get("endtime");
//rangbuilder.lte(endtime+",000");
queryMap.remove("endtime");
rangbuilder.to(endtime+",000");
if(!ifhasstar) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
String starttime = df.format(new Date(0));
//rangbuilder.gte(starttime+",000");
rangbuilder.from(starttime);
}else {
String starttime = (String) queryMap.get("starttime")+",000";
rangbuilder.from(starttime);
queryMap.remove("starttime");
}
//sourceBuilder.query(rangbuilder);
boolQueryBuilder.must(rangbuilder);
}else {
if(ifhasstar) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
String endtime = df.format(new Date());
//rangbuilder.lte(endtime+",000");
rangbuilder.to(endtime);
String starttime = (String) queryMap.get("starttime")+",000";
rangbuilder.from(starttime);
//sourceBuilder.query(rangbuilder);
boolQueryBuilder.must(rangbuilder);
queryMap.remove("starttime");
}else {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
String endtime = df.format(new Date());
rangbuilder.to(endtime);
String starttime = df.format(new Date(0));
rangbuilder.from(starttime);
}
}
// boolQueryBuilder.must(rangbuilder);
Set keyset = queryMap.keySet();
Iterator keyit = keyset.iterator();
while(keyit.hasNext()) {
String nowkey = keyit.next();
String str = (String)queryMap.get(nowkey);//得到每个key多对用value的值
if(str != null && !"".equals(str)) {
//boolQueryBuilder.must(QueryBuilders.matchQuery(nowkey,str).minimumShouldMatch("80%"));
boolQueryBuilder.must(QueryBuilders.wildcardQuery(nowkey,"*"+str+"*" ));
}
}
QueryBuilder queryBuilder = boolQueryBuilder;
//分页查看功能
sourceBuilder.query(queryBuilder);
//后续处理 应该是动态的
sourceBuilder.from((nowpage-1) * 10);
if(nowpage == 1000) {
showsize = 10;
}
sourceBuilder.size(showsize);
sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
//根据日期进行排序
FieldSortBuilder fsb = SortBuilders.fieldSort("date.keyword");
if("ASC".equals(sortway)) {
fsb.order(SortOrder.ASC);
}else {
fsb.order(SortOrder.DESC);
}
sourceBuilder.sort(fsb);
//sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
//将搜索条件加载到searchRequest中
searchRequest.source(sourceBuilder);
//searchRequest.types("log");
//执行查询操作
//优化 避免将错误抛到页面 进行try-catch
SearchResponse searchResponse =null;
try {
searchResponse = client.search(searchRequest);
} catch (Exception e) {
// TODO: handle exception
logger.debug("===================查询出错了。");
return result;
}
//SearchResponse searchResponse = client.search(searchRequest);
SearchHits hits = searchResponse.getHits();
long totalHits = hits.getTotalHits();
logger.debug("=============查询到的数据有"+totalHits);
SearchHit[] searchHits = hits.getHits();
List> loglist = new ArrayList>();
Map yyToFbsjq = queryFbsJqIpByYhuuid(yhuuid);
if(ifshowAll != null && ifshowAll.equals("Y")) {
for (SearchHit hit : searchHits) {
/*String type = hit.getType();
String name = hit.getIndex();*/
Map tmp = hit.getSourceAsMap();
/* String message = (String) tmp.get("message");
System.out.println("message内容是" + message);
System.out.println("类型是："+ type);
System.out.println("Index 名称是："+ name);*/
@SuppressWarnings("unchecked")
Map yynamemap = (HashMap)tmp.get("fields");
String content = (String) tmp.get("message");
if(content.contains("\n")) {
content = content.replace("\n","
");
tmp.put("message", content);
}
String yyname = yynamemap.get("yyname");
if(yyToFbsjq != null ) {
String fbsurl = "http://" + yyToFbsjq.get(yyname)+":8980/#/trace?traceId=";
tmp.put("fbsurl",fbsurl);
}
tmp.put("yyname",yyname);
loglist.add(tmp);
}
}else {
for (SearchHit hit : searchHits) {
Map tmp = hit.getSourceAsMap();
@SuppressWarnings("unchecked")
Map yynamemap = (HashMap)tmp.get("fields");
String yyname = yynamemap.get("yyname");
if(yyToFbsjq != null ) {
String fbsurl = "http://" + yyToFbsjq.get(yyname)+":8980/#/trace?traceId=";
tmp.put("fbsurl",fbsurl);
}
tmp.put("yyname",yyname);
loglist.add(tmp);
}
}