SpringBoot 集成ElasticSearch的几种方式

SpringBoot 集成ElasticSearch的几种方式

 

 SpringBoot 集成ElasticSearch有两种方式，

1）通过客户端 如JestClient 。优点：依赖包少。

2) spring-data框架。优点：可应用框架提供的功能。

 

准备工作：docker安装elasticSearch、kibana

elasticSearch、kibana版本选取7.9.0，两个版本需要一样

docker pull elasticsearch:7.9.0

# 创建网络 将es、kibana放在同一个网络中

kibana可以简单的使用127.0.0.1访问es,否则需要docker inspect elasticsearch 查看es的docker ip。

docker network create esnet

# 启动elasticsearch --net esnet 指定网络

docker run --name elasticsearch --net esnet -d -e ES_JAVA_OPTS="-Xms256m -Xmx1024m" -e "discovery.type=single-node" -p 9200:9200 -p 9300:9300 elasticsearch:7.9.0

#启动kibana  ELASTICSEARCH_URL 指定链接的ES

docker run --name kibana --net esnet -e ELASTICSEARCH_URL=http://127.0.0.1:9200 -p 5601:5601 -d kibana:7.9.0

# localhost 替换为docker的ip。docker 在本机windows、mac上安装为localhost，在VirtualBox上安装为虚拟机的IP

elasticsearch: http://localhost:9200/

kaibana：http://localhost:5601/

# 数据准备工作，所有操作都可以用jest完成，本文针对已有数据场景，只做查询操作

kibana的devTool

创建索引，模拟日志索引 按天 logs-2021.05.25

PUT /logs-2021.05.25/

创建索引映射 _mapping 。可以跳过，使用ES动态mapping

POST /logs-2021.05.24/_mapping{      "dynamic" : "true",      "_source" : {        "includes" : [ ],        "excludes" : [ ]      },      "dynamic_date_formats" : [        "strict_date_optional_time",        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"      ],      "date_detection" : true,      "numeric_detection" : false,      "properties" : {        "@timestamp" : {          "type" : "date"        },        "@version" : {          "type" : "long"        },        "appName" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "class" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "env" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "error" : {          "properties" : {            "id" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            }          }        },        "exception" : {          "properties" : {            "exception_class" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            },            "exception_message" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            },            "stacktrace" : {              "type" : "text"            }          }        },        "file" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "git" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "level" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "line_number" : {          "type" : "long"        },        "logger_name" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "mdc" : {          "properties" : {            "depth" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            },            "perm_id" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            }          }        },        "message" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "method" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "source_host" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "tag" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "thread_name" : {          "type" : "text",          "fields" : {            "keyword" : {              "type" : "keyword",              "ignore_above" : 256            }          }        },        "trace" : {          "properties" : {            "id" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            }          }        },        "transaction" : {          "properties" : {            "id" : {              "type" : "text",              "fields" : {                "keyword" : {                  "type" : "keyword",                  "ignore_above" : 256                }              }            }          }        }      }    }

查看创建的映射

•  

GET /logs-2021.05.25/_mapping

插入数据

POST /logs-2021.05.25/_doc/{    "class" : "",    "appName" : "activity",    "source_host" : "node1-dev",    "trace.id" : "abcef",    "@timestamp" : "2021-05-25T10:10:00.101Z",    "thread_name" : "http-nio-22500-exec-01",    "level" : "ERROR",    "@version" : 1,    "env" : "prod",    "git" : "b64dc78",    "transaction.id" : "16cb1b2d8d888",    "message" : "活动数据",    "method" : "send",    "line_number" : 16,    "file" : "SendServiceImpl.java",    "logger_name" : "com.xxx",    "tag" : "git-tag-122_2",    "mdc" : { }  }

查询数据， 客户端jest中可以直接应用kibana中的query查询语句

GET /logs-2021.05.25/_search{  "query": {    "match_all": {}  }}

至此数据准备完成。

 一、通过客户端 JestClient 连接 （其他客户端 RestHighLevelClient推荐使用）

 

版本选择：maven 搜索jest,最新版本6.3.1，适应es版本 6.3.1--7.12.1

 

    io.searchbox    jest    6.3.1

 

pom依赖

未指定版本在父pom中指定，工具类版本可随意指定。

      org.springframework.boot      spring-boot-starter-web      2.2.5.RELEASE    
          com.alibaba      fastjson    
          org.projectlombok      lombok    
          cn.hutool      hutool-all    
          com.github.xiaoymin      knife4j-spring-boot-starter      3.0.2    
              io.searchbox      jest      6.3.1    

application.yml配置连接地址，es未开启认证用户密码不用填写

spring:  elasticsearch:    jest:      uris: http://localhost:9200#      username: your_es_username#      password: youer_es_password

用Controller做测试，或者写Test

@Slf4j@RestController@Api(tags = "ES管理器")public class EsController {
  final String indexName ="logs-2021.05.24";//  final String indexName ="logs-*";
  final String INFO = "info";  final String WARN = "warn";  final String ERROR = "error";
  @Autowired  private JestClient jestClient;

  @ApiOperation("值班日志--活动")  @GetMapping("/search")  public Object search(){    String appName = "product-api";    String level="ERROR";
    String indexName = "logs-2021.05.25";        // kibana中的查询语句直接应用 bool、filter、term、match、match_phrase    String query1 = "{\n"    + "  \"query\": {\n"    + "    \"match_all\": {}\n"    + "  }\n"    + "}";
    // #此查询指定了大小size  筛选了appName=activity  level=ERROR的数据    String query = "{\n"        + "  \"from\":0,\"size\":10000,\n"        + "  \"query\": {\n"        + "    \"bool\": {\n"        + "      \"filter\": [\n"        + "          {\n"        + "            \"term\": {\n"        + "              \"appName.keyword\": {\n"        + "                \"value\": \"activity\"\n"        + "              }\n"        + "            }\n"        + "        },\n"        + "        {\n"        + "          \"term\": {\n"        + "            \"level.keyword\": {\n"        + "              \"value\": \"ERROR\"\n"        + "            }\n"        + "          }\n"        + "        }\n"        + "      ]\n"        + "    }\n"        + "  }\n"        + "}";               String result = "success";    try {      Search search = new Search.Builder(query).addIndex(indexName).addType("_doc").build();      JestResult jr = jestClient.execute(search);      List sourceAsObjectList = jr.getSourceAsObjectList(DayLog.class);      System.out.println("--++"+jr.getJsonString());      System.out.println("--++"+sourceAsObjectList.size());      result = jr.getJsonString();    } catch (IOException e) {      e.printStackTrace();    }    return result;  }}

引入elasticsearch包通过工具包提供的方法构建语句。如通过QueryBuilders构建查询。版本跟ES版本一致。

  org.elasticsearch  elasticsearch  7.9.0

上面的query查询语句可以如下构建，可以看到searchSourceBuilder.toString()跟kibana中的语句是一样的。

String appName = "activity";String level= "ERROR";
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();      searchSourceBuilder.query(QueryBuilders.boolQuery()          .filter(QueryBuilders.termQuery("appName.keyword",appName))          .filter(QueryBuilders.termQuery("level.keyword",level)));      searchSourceBuilder.from(0).size(10000);
      System.out.println("query:\n"+searchSourceBuilder.toString());      Search search =new Search.Builder(searchSourceBuilder.toString()).addIndex(indexName).addType("_doc").build();      JestResult jestResult = jestClient.execute(search);      JsonObject resultJSON = jestResult.getJsonObject();      JsonObject totalJson = resultJSON.getAsJsonObject("hits").getAsJsonObject("total");      long totalNum = totalJson.get("value").getAsLong();      String relation = totalJson.get("relation").getAsString();
      System.out.println("=============== "+jestResult.getJsonString());

jestClient的查询文档介绍到此，还有更多的应用参考网络。

 

 

二、Spring-Data 集成ElasticSearch

选择合适的版本，pom 查看spring-boot-dependencies  2.2.5.RELEASE 默认支持ES版本是6.8.6

 

maven中查看支持es版本6.8.6--7.12.1，ES7.0以后改动较大，考虑切换版本。

 

最新版本 2.5.0 已支持7.12.1

经maven查询spring-boot-dependencies 版本2.3.x支持 spring-data-elsaticsearch 4.0.x ES最低版本7.6.2，最终选取2.3.5.RELEASE

pom 依赖

                org.springframework.boot      spring-boot-dependencies      2.3.5.RELEASE      pom      import      

      org.springframework.boot    spring-boot-starter-data-elasticsearch  

 

栗子

@Slf4j@Api(tags = "ES搜索")@RestControllerpublic class DayLogController {
  @Autowired  private ElasticsearchRestTemplate restTemplate;
  @ApiOperation("search")  @GetMapping("/search")  public Object search(){
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();    boolQueryBuilder        .filter(QueryBuilders.termQuery("appName.keyword","activity"))        .filter(QueryBuilders.termQuery("level.keyword","ERROR"));
    NativeSearchQuery searchQuery = new NativeSearchQuery(boolQueryBuilder);    log.info("query: {}",searchQuery.getQuery().toString());    //索引名称规定 可在类上标注，ES会根据类标注生成索引IndexCoordinates    SearchHits searchHits0 = restTemplate.search(searchQuery, DayLog.class);    //索引不固定 以参数方式传入    String indexName = "logs-2021.05.25";    SearchHits searchHits = restTemplate.search(searchQuery, DayLog.class, IndexCoordinates.of(indexName));    System.out.println(JSON.toJSONString(searchHits));
    return JSON.toJSONString(searchHits);  }
}DayLogs.java@Data@Document(indexName = "logs-2021.05.25")public class DayLog {
  @Field(name = "appName", type = FieldType.Keyword)  private String appName;
  @Field(name = "source_host",type = FieldType.Text)  private String sourceHost;

  private String timestamp;
  @Field(name = "thread_name",type = FieldType.Text)  private String threadName;
  private String env;
    @JSONField(name = "class")  @Field(name = "class",type = FieldType.Text)  private String klass;
  private String error;
  private String file;
  private String git;
  @Field(name = "level", type = FieldType.Keyword)  private String level;
  private String message;
  private String method;
  @Field(name = "line_number",type = FieldType.Long)  private Long lineNumber;
  @Field(name = "logger_name",type = FieldType.Text)  private String loggerName;
  private String tag;
}

简单的代码即可实现查询。通过NativeSearchQuery构建查询语句，输出如下标准语句，ElasticsearchRestTemplate对RestHighLevelClient客户端进行了封装，应用起来更方便。

# NativeSearchQuery构建查询语句query: {  "bool" : {    "filter" : [      {        "term" : {          "appName.keyword" : {            "value" : "activity",            "boost" : 1.0          }        }      },      {        "term" : {          "level.keyword" : {            "value" : "ERROR",            "boost" : 1.0          }        }      }    ],    "adjust_pure_negative" : true,    "boost" : 1.0  }}

 

三、RestHighLevelClient  客户端

2.2.5.RELEASE 默认ES版本6.8.6，方式一：升级Spring-boot-dependencies 到2.3.x，方式二：升级ES

升级版本 2.3.5.RELEASE

            org.springframework.boot      spring-boot-dependencies      2.3.5.RELEASE      pom      import      引入依赖pom  org.elasticsearch.client  elasticsearch-rest-high-level-client

查询示例

@Api(tags = "RestHighLevel客户端")@RestControllerpublic class EsRestHighLevelController {
  @Autowired  private RestHighLevelClient restHighLevelClient;
  @ApiOperation("search")  @GetMapping("/search")  public Object search(){
    SearchRequest searchRequest = new SearchRequest("logs-2021.05.25");    // 构造条件    SearchSourceBuilder builder = new   SearchSourceBuilder();
    // 精确匹配    TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("appName.keyword","activity");    TermQueryBuilder termQueryBuilder2 = QueryBuilders.termQuery("level.keyword","ERROR");    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().filter(termQueryBuilder).filter(termQueryBuilder2);    builder.query(boolQueryBuilder);    // 分页    builder.from();    builder.size(100);    builder.timeout(new TimeValue(60, TimeUnit.SECONDS));    searchRequest.source(builder);    String result = "";    try {      SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);      result = JSON.toJSONString(response);      System.out.println(JSON.toJSONString(response.getHits()));    } catch (IOException e) {      e.printStackTrace();    }
    return result;  }}

RestHighLevel简单的查询功能已完成。更多功能查看RestHighLevel的doc。

总结：

连接ES通常用客户端方式 jest、RestHeighLevel，Spring Boot集成spring-data-elasticsearch方式，ElasticsearchRestTemplate 进行了一些封装，对类型转换较为方便。

 

modules:  

        paw-jest        paw-es7        paw-ESRestHighLevel

gitee: https://gitee.com/tg_seahorse/paw-demos

 

 

 

喜欢就转发+关注下吧