ELK高级搜索三Spring boot 接入Elasticsearch
ES数据写入路由机制
索引是分片设计，路由选择写入到那个分片，优先写入到主分片，再次写入到副本分片。路由计算默认基于数据ID，也可以自己指定。
数据先写入Buffer，同时写入Translog（用于极端情况下的数据恢复），Buffer缓存数据达到阈值会批量刷到磁盘（中间有个文件系统缓冲），所以说es的数据写入是一个近实时的（延时默认是1秒）
数据删除内部机制
ES采用标记删除，不会真正意义上马上物理删除，待下次写入到磁盘才会真正删除
Java api 实现文档管理
es技术比较特殊，不像其他分布式、大数据课程，haddop、spark、hbase。es代码层面很好写，难的是概念的理解。最重要的是他的rest api。跨语言的。在真实生产中，探查数据、分析数据，使用rest更方便。
maven依赖
org.elasticsearch.client
elasticsearch-rest-high-level-client
7.10.1
org.elasticsearch
elasticsearch
org.elasticsearch
elasticsearch
7.10.1
创建索引
使用kibana 创建索引，并手动添加几条数据，方便后面测试。索引的名称：article ，别名：news。
PUT /article
{
"settings":{
"number_of_shards":3,
"number_of_replicas":2
},
"mappings":{
"dynamic":false,
"properties":{
"title":{
"type":"text",
"analyzer":"ik_smart",
"search_analyzer":"ik_max_word"
},
"content":{
"type":"text",
"analyzer":"ik_smart",
"search_analyzer":"ik_max_word"
},
"categoryName":{
"type":"keyword"
},
"view_count":{
"type": "integer"
},
"publishTime":{
"type":"date",
"format":"yyyy-MM-dd HH:mm:ss||yyyy-MM-dd"
}
}
},
"aliases":{
"news":{
}
}
}
ES API的操作步骤
- 获取连接客户端
- 构建请求
- 执行请求
-
获取结果
获取连接
@SpringBootTest(classes = WebApplication.class)
@RunWith(SpringRunner.class)
public class ElasticsearchTest {
private RestHighLevelClient client;
/**
* 初始化客户端
*/
@Before
public void init() {
//获取连接客户端
client = new RestHighLevelClient(RestClient.builder(new HttpHost("127.0.0.1", 9200, "http")));
}
/**
* 销毁链接
*/
@After
public void destory() {
if (client != null) {
try {
client.close();
} catch (Exception ex) {
}
}
}
}查询文档测试
/**
* 根据文档ID查询
*/
@Test
public void tetGetDoc() throws Exception {
// 构建请求
GetRequest getRequest = new GetRequest("news", "1");
// 执行请求
GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
if (getResponse != null && getResponse.isExists()) {
String sourceAsString = getResponse.getSourceAsString();
// 获取结果
System.out.println(sourceAsString);
} else {
System.out.println("查询异常");
}
}输出结果
{
"title":"芯片人才争夺",
"content":"芯片人才争夺“生猛”,需要大量人才",
"categoryName":"科技",
"view_count":60,
"publishTime":"2022-04-19 12:00:00"
}异步查询文档测试
/**
* 异步根据文档ID查询
*/
@Test
public void tetAsyncGetDoc() throws Exception {
// 构建请求
GetRequest getRequest = new GetRequest("news", "1");
// 执行请求
client.getAsync(getRequest, RequestOptions.DEFAULT, new ActionListener() {
@Override
public void onResponse(GetResponse getResponse) {
if(getResponse != null && getResponse.isExists()) {
// 获取结果
System.out.println(JSON.toJSONString(getResponse));
} else {
System.out.println("查询异常");
}
}
@Override
public void onFailure(Exception e) {
System.out.println(e);
}
});
try {
Thread.sleep(4000);
} catch (Exception ex) {
}
} 输出结果
{
"exists":true,
"fields":{
},
"fragment":false,
"id":"1",
"index":"article",
"primaryTerm":1,
"seqNo":0,
"source":{
"publishTime":"2022-04-19 12:00:00",
"title":"芯片人才争夺",
"categoryName":"科技",
"content":"芯片人才争夺“生猛”,需要大量人才",
"view_count":60
},
"sourceAsBytes":"eyJ0aXRsZSI6IuiKr+eJh+S6uuaJjeS6ieWkuiIsImNvbnRlbnQiOiLoiq/niYfkurrmiY3kuonlpLrigJznlJ/njJvigJ0s6ZyA6KaB5aSn6YeP5Lq65omNIiwiY2F0ZWdvcnlOYW1lIjoi56eR5oqAIiwidmlld19jb3VudCI6NjAsInB1Ymxpc2hUaW1lIjoiMjAyMi0wNC0xOSAxMjowMDowMCJ9",
"sourceAsBytesRef":{
"fragment":true
},
"sourceAsMap":{
"$ref":"$.source"
},
"sourceAsString":"{\"title\":\"芯片人才争夺\",\"content\":\"芯片人才争夺“生猛”,需要大量人才\",\"categoryName\":\"科技\",\"view_count\":60,\"publishTime\":\"2022-04-19 12:00:00\"}",
"sourceEmpty":false,
"sourceInternal":{
"$ref":"$.sourceAsBytesRef"
},
"type":"_doc",
"version":1
}创建文档测试
es节点所在的磁盘空间不够用的时候，es会将该节点上面的索引标位只读，不能向该索引写入数据，默认当磁盘空间超过85%，我们可以使用下面的命令调整磁盘空间比例为95%
PUT localhost:9200/_cluster/settings
{
"transient": {
"cluster.routing.allocation.disk.watermark.low": "90%",
"cluster.routing.allocation.disk.watermark.high": "95%",
"cluster.info.update.interval": "1m"
}
} /**
* 创建文档
*/
@Test
public void testCreate() throws Exception {
Map jsonMap = new HashMap<>();
jsonMap.put("title", "今日头条、小红书等多家互联网平台将显示IP属地");
jsonMap.put("content", "今日头条、小红书等多家互联网平台将显示IP属地");
jsonMap.put("categoryName", "科技");
jsonMap.put("view_count", 77);
jsonMap.put("publishTime", "2022-04-18");
IndexRequest request = new IndexRequest("news")
.id(String.valueOf(5))
.source(jsonMap)
.opType(DocWriteRequest.OpType.CREATE);
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
if (response == null || !Objects.equals(response.getResult(), DocWriteResponse.Result.CREATED)) {
System.out.println("创建异常");
} else {
System.out.println(JSON.toJSONString(response));
}
} 输出结果
{
"fragment":false,
"id":"5",
"index":"article",
"primaryTerm":1,
"result":"CREATED",
"seqNo":0,
"shardId":{
"fragment":true,
"id":-1,
"index":{
"fragment":false,
"name":"article",
"uUID":"_na_"
},
"indexName":"article"
},
"shardInfo":{
"failed":0,
"failures":[
],
"fragment":false,
"successful":1,
"total":3
},
"type":"_doc",
"version":1
}异步创建文档
/**
* 异步创建文档
*/
@Test
public void testAsyncAdd() throws Exception {
Map jsonMap = new HashMap<>();
jsonMap.put("title", "微信朋友圈上线十周年 还记得你第一条发的什么内容吗？");
jsonMap.put("content", "微信朋友圈上线十周年 还记得你第一条发的什么内容吗？");
jsonMap.put("categoryName", "科技");
jsonMap.put("view_count", 88);
jsonMap.put("publishTime", "2022-04-19");
IndexRequest request = new IndexRequest("news")
.id("6")
.source(jsonMap);
client.indexAsync(request, RequestOptions.DEFAULT, new ActionListener() {
@Override
public void onResponse(IndexResponse response) {
if (response != null && Objects.equals(response.getResult(), DocWriteResponse.Result.CREATED)) {
System.out.println(JSON.toJSONString(response));
} else {
System.out.println("创建异常");
}
}
@Override
public void onFailure(Exception e) {
System.out.println(e);
}
});
try {
Thread.sleep(4000);
} catch (Exception ex) {
}
} 编辑文档
/**
* 部分更新
*/
@Test
public void testUpdate() throws Exception {
Map jsonMap = new HashMap<>();
jsonMap.put("content", "微信朋友圈上线十周年 第一条发的什么内容呢？你知道吗");
UpdateRequest request = new UpdateRequest("news", "5").doc(jsonMap);
UpdateResponse response = client.update(request, RequestOptions.DEFAULT);
if (response != null && Objects.equals(response.getResult(), DocWriteResponse.Result.UPDATED)) {
System.out.println(JSON.toJSONString(response));
} else {
System.out.println("修改异常");
}
} 输出结果：
{
"fragment":false,
"id":"5",
"index":"article",
"primaryTerm":1,
"result":"UPDATED",
"seqNo":1,
"shardId":{
"fragment":true,
"id":-1,
"index":{
"fragment":false,
"name":"article",
"uUID":"_na_"
},
"indexName":"article"
},
"shardInfo":{
"failed":0,
"failures":[
],
"fragment":false,
"successful":1,
"total":3
},
"type":"_doc",
"version":2
}删除文档
/**
* 根据文档ID删除
*/
@Test
public void testDelete() throws Exception {
DeleteRequest request = new DeleteRequest("news", "5");
DeleteResponse response = client.delete(request, RequestOptions.DEFAULT);
if (response != null && Objects.equals(DocWriteResponse.Result.DELETED, response.getResult())) {
System.out.println(JSON.toJSONString(response));
} else {
System.out.println("删除异常");
}
}输出结果：
{
"fragment":false,
"id":"5",
"index":"article",
"primaryTerm":1,
"result":"DELETED",
"seqNo":2,
"shardId":{
"fragment":true,
"id":-1,
"index":{
"fragment":false,
"name":"article",
"uUID":"_na_"
},
"indexName":"article"
},
"shardInfo":{
"failed":0,
"failures":[
],
"fragment":false,
"successful":1,
"total":3
},
"type":"_doc",
"version":3
}
创建索引
/**
* 创建索引
*/
@Test
public void testCreateIndex() throws Exception {
//创建索引对象
CreateIndexRequest request = new CreateIndexRequest("test_index");
//设置参数
request.settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0));
Map titleMap = Maps.newHashMap();
titleMap.put("type", "text");
Map properties = Maps.newHashMap();
properties.put("title", titleMap);
Map mapping = Maps.newHashMap();
mapping.put("properties", properties);
//指定映射
request.mapping(mapping);
//指定别名
request.alias(new Alias("test_index_alias"));
IndicesClient indicesClient = client.indices();
CreateIndexResponse response = indicesClient.create(request, RequestOptions.DEFAULT);
System.out.println(JSON.toJSONString(response));
} 输出：{"acknowledged":true,"fragment":false,"shardsAcknowledged":true}
异步创建索引
/**
* 创建索引
*/
@Test
public void testAsyncCreateIndex() throws Exception {
//创建索引对象
CreateIndexRequest request = new CreateIndexRequest("test_index_async");
//设置参数
request.settings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0));
Map titleMap = Maps.newHashMap();
titleMap.put("type", "text");
Map properties = Maps.newHashMap();
properties.put("title", titleMap);
Map mapping = Maps.newHashMap();
mapping.put("properties", properties);
//指定映射
request.mapping(mapping);
//指定别名
request.alias(new Alias("test_index_async_alias"));
IndicesClient indicesClient = client.indices();
Cancellable cancellable = indicesClient.createAsync(request, RequestOptions.DEFAULT, new ActionListener() {
@Override
public void onResponse(CreateIndexResponse response) {
System.out.println(JSON.toJSONString(response));
}
@Override
public void onFailure(Exception e) {
System.out.println(e);
}
});
try {
Thread.sleep(4000);
} catch (Exception ex) {
}
} 输出：{"acknowledged":true,"fragment":false,"shardsAcknowledged":true}
索引是否存在
/**
* 索引是否存在
*/
@Test
public void testExistsIndex() throws Exception {
GetIndexRequest request = new GetIndexRequest("test_index");
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
System.out.println(exists);
}输出：true
删除索引
/**
* 删除索引
*/
@Test
public void testDeleteIndex() throws Exception {
DeleteIndexRequest request = new DeleteIndexRequest("test_index");
AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT);
System.out.println(JSON.toJSONString(response));
}输出：{"acknowledged":true,"fragment":false}