docker-compose搭建ElasticSearch7.4.0集群+kibana
配置文件结构图(配置详见本文最下方)
docker-compose 运行起来后,由于开启了xpack.security,需要先生成证书
docker run -dit --name=es elasticsearch:7.4.0 /bin/bash //临时运行一个实例
docker exec -it es /bin/bash //进入容器
./bin/elasticsearch-certutil ca
./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 //生成证书
cp es:证书目录 宿主机存放路径 //复制证书到宿主机
docker exec -it elastic01 /bin/bas
./bin/elasticsearch-setup-passwords -h
./bin/elasticsearch-setup-passwords auto //随机生成密码用auto, 自己设置用 interactive
Changed password for user apm_system
PASSWORD apm_system = L52hnqb5NWFfz9p43pl6
Changed password for user kibana
PASSWORD kibana = 8cidPDyFc1K6kHyRyFMq
Changed password for user logstash_system
PASSWORD logstash_system = 7yb9Ss1z3vg4omidXqTa
Changed password for user beats_system
PASSWORD beats_system = 7EFxeC8U0uWDNHsT8eTD
Changed password for user remote_monitoring_user
PASSWORD remote_monitoring_user = bp4JQx5P98ZsksDphvuv
Changed password for user elastic
PASSWORD elastic = ZJDa3VAtxDNZrXuEBYTS
然后更改kibana.yml配置文件的密码,账号为:kibana
docker-compose up -d
F:\work\docker-ElasticSearch-7.4.0>docker exec -it elastic02 /bin/bash
[root@f83bef560f84 elasticsearch]# curl -u elastic:ZJDa3VAtxDNZrXuEBYTS elastic01:9200/_cluster/health?pretty
{
"cluster_name" : "es-docker-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 20,
"active_shards" : 40,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
这个问题是在启动kibana的时候出现的,完全启动不起来,而且cpu直接100%,后面去看了docker 日志发现的问题
max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
官方提供了解决办法:
sysctl -w vm.max_map_count=262144
wsl -d docker-desktop sysctl -w vm.max_map_count=262144
org.springframework.boot
spring-boot-starter-data-elasticsearch
com.fasterxml.jackson.dataformat
jackson-dataformat-smile
com.fasterxml.jackson.dataformat
jackson-dataformat-cbor
net.sf.jopt-simple
jopt-simple
org.projectlombok
lombok
1.18.20
com.alibaba
fastjson
1.2.46
cn.hutool
hutool-all
5.5.8
application.yml配置文件
server:
port: 16102
spring:
application:
name: elastic-service
logging:
level:
cn.laisanjin.elasticsearch: info
elasticsearch:
uris: 192.168.0.185:9201
username: elastic
password: ZJDa3VAtxDNZrXuEBYTS
ElasticsearchConfig配置文件
@Component
@Configuration
@ConfigurationProperties(prefix = "spring.elasticsearch.rest")
public class ElasticsearchConfig extends AbstractElasticsearchConfiguration {
@Value("${elasticsearch.uris}")
private String uris;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@Override
@Bean(destroyMethod = "close")
public RestHighLevelClient elasticsearchClient() {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(uris)
.withBasicAuth(username, password)
.build();
return RestClients.create(clientConfiguration).rest();
}
}
@Autowired
private RestHighLevelClient restHighLevelClient;
public void createIndex(String index) throws Exception {
boolean product = restHighLevelClient.indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT);
if (product == false) {
CreateIndexRequest request = new CreateIndexRequest(index);
CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
} else {
System.out.println("index:" + index + " exist");
}
}
@Autowired
private RestHighLevelClient restHighLevelClient;
void deleteIndex(String index) throws Exception {
AcknowledgedResponse product = restHighLevelClient.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT);
}
public void insertDoc(String index, String jsonObject, String id) throws Exception {
//创建请求体
IndexRequest request = new IndexRequest(index);
request.id(id);
request.source(jsonObject, XContentType.JSON);
//插入一条文档
IndexResponse response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
}
public void batchInsertDoc(ArrayList arrayList) throws Exception {
BulkRequest bulkRequest = new BulkRequest();
for (JSONObject jsonObject : arrayList) {
String index = jsonObject.getString("index");
String id = jsonObject.getString("docId");
JSONObject data = JSONObject.parseObject(jsonObject.getString("data"));
bulkRequest.add(new IndexRequest().index(index).id(id).source(data, XContentType.JSON));
}
BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
public JSONObject searchPage(String index, JSONObject jsonObject, int page, int pageSize) throws Exception {
SearchRequest searchRequest = new SearchRequest();
//设置索引
searchRequest.indices(index);
SearchSourceBuilder builder = new SearchSourceBuilder();
if (ObjectUtil.isNotEmpty(jsonObject)) {
//设置搜索字段
if (jsonObject.containsKey("selectColumns") && StrUtil.isNotEmpty(jsonObject.getString("selectColumns"))) {
builder.fetchSource(jsonObject.getString("selectColumns").split(","), null
);
}
//组装搜索条件
if (jsonObject.containsKey("searchParams") && StrUtil.isNotEmpty(jsonObject.getString("searchParams"))) {
List conditionList = JSONObject.parseArray(jsonObject.getString("searchParams"), JSONObject.class);
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
for (JSONObject conditionJson : conditionList) {
if (
!conditionJson.containsKey("condition")
|| !conditionJson.containsKey("field")
|| !conditionJson.containsKey("value")
) {
continue;
}
String condition = conditionJson.getString("condition");
String field = conditionJson.getString("field");
String value = conditionJson.getString("value");
if (condition.equals("eq")) {//等于
boolQueryBuilder.must(QueryBuilders.termQuery(field, value));
} else if (condition.equals("like")) {//模糊搜索
boolQueryBuilder.must(QueryBuilders.fuzzyQuery(field,value));
} else if (condition.equals("gt")) {//大于
boolQueryBuilder.must(QueryBuilders.rangeQuery(field).gt(value));
} else if (condition.equals("lt")) {//小于
boolQueryBuilder.must(QueryBuilders.rangeQuery(field).lt(value));
} else if (condition.equals("in")) {
String[] strings = value.split(",");
BoolQueryBuilder boolQueryBuilderIn = QueryBuilders.boolQuery();
for (String val : strings) {
boolQueryBuilderIn.should(QueryBuilders.termQuery(field, val));
}
boolQueryBuilder.must(boolQueryBuilderIn);
} else if (condition.equals("notIn")) {
String[] strings = value.split(",");
BoolQueryBuilder boolQueryBuilderIn = QueryBuilders.boolQuery();
for (String val : strings) {
boolQueryBuilderIn.should(QueryBuilders.termQuery(field, val));
}
boolQueryBuilder.mustNot(boolQueryBuilderIn);
}
}
builder.query(boolQueryBuilder);
}
//排序
if (jsonObject.containsKey("sortOrder") && StrUtil.isNotEmpty(jsonObject.getString("sortOrder"))) {
List sortJsonList = JSONObject.parseArray(jsonObject.getString("sortOrder"), JSONObject.class);
for (JSONObject sortJson : sortJsonList) {
SortOrder sortOrder = SortOrder.DESC;
if (sortJson.getString("sort").equals("asc")) {
sortOrder = SortOrder.ASC;
}
builder.sort(sortJson.getString("filed"), sortOrder);
}
}
}
System.out.println("builder:" + builder);
//页码
builder.from(page);
//条数
builder.size(pageSize);
searchRequest.source(builder);
//查询主体
SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = response.getHits();
//返回结果
JSONObject responseJson = new JSONObject();
responseJson.put("total", hits.getTotalHits().value);
responseJson.put("took", response.getTook().getStringRep());
responseJson.put("currentPage", page);
boolean isMore = false;
if (hits.getTotalHits().value > page * pageSize) {
isMore = true;
}
responseJson.put("isMore", isMore);
JSONArray dataList = new JSONArray();
for (SearchHit hit : hits) {
dataList.add(JSONObject.parseObject(hit.getSourceAsString()));
}
responseJson.put("currentTotal", dataList.size());
responseJson.put("list", dataList);
return responseJson;
}
(备注:可以使用compose安装)
以下采用curl http请求
注意,因为开启了证书,请求Url格式为:http://elastic账号:elastic密码@elastic请求路径
例如:http://elastic:[email protected]:9201
以下案例直接用kibana console做演示
{
"mappings": {
"properties": {
"ID": {
"type": "keyword",
"index": true
},
"sName": {
"type": "text",
"index": true
},
"bActive": {
"type": "integer",
"index": true,
"null_value": 0
}
}
}
}
Array
(
[lID] => 55
[sName] => 你好怀念のXO酱罐装 台湾经典素酱 家喻户晓
[fPrice] => 38.00
[ProductCatID] => 74
)
新建文档(post)
指定ID:/product/_doc/55
自动生成ID请求url:/product/_doc?pretty
文档搜索(post)
请求url:/product/_search
参数:
{
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"match_phrase": {
"sName": "预制菜"
}
},
{
"match_phrase": {
"productCatName": "预制菜"
}
}
]
}
},
{
"range": {
"fPrice": {
"gt": 0
}
}
},
{
"term": {
"bSale": 1
}
}
]
}
},
"sort": {
"dEditDate": {
"order": "desc"
}
},
"_source": [
"lID",
"sName",
"ProductCatID",
"fPrice",
"SaasUserID",
"bDel",
"dEditDate",
"bSale",
"lStock",
"productCatName"
],
"from": 0,
"size": 10
}
参数说明
* elasticsearch与mysql做比较
* _source=需要搜索的字段
*
* must=and
* mysql eq:(bSale=1 and bDel=0)
*
* should=or
* mysql eq:(bSale=1 or bDel=0)
*
* match=like,match=分词模糊搜索 match_phrase=不分词模糊搜索
* mysql eq:(sName like "%水果%")
*
* term=精确搜索
* mysql eq:(SaasUserID=1)
*
* range=范围搜索
* mysql eq:(fPrice>0)
*
* sort = 排序
* mysql eq:(order by fPrice desc)
*
* from=页码 elastic默认是0开始
* size=条数 elastic默认是10
public static function sendPost($url, $post)
{
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
curl_setopt($ch, CONNECTION_TIMEOUT, 60);
$data = curl_exec($ch);
curl_close($ch);
return $data;
}
public static function sendPut(string $url, $params = [], $headers = [])
{
$timeout = 5;
$headers = $headers ?: array('Content-type: application/json');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url); //发贴地址
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "PUT");
curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
$response = curl_exec($ch);//获得返回值
curl_close($ch);// 关闭 cURL 释放资源
if ($error = curl_error($ch)) {
$result = array(
'status' => false,
'message' => $error,
);
} else {
$result = array(
'status' => true,
'message' => 'ok',
'resp' => json_decode($response, true),
);
}
return $result;
}
public static function sendDelete(string $url, $params = [], $headers = [])
{
$timeout = 5;
$headers = $headers ?: array('Content-type: text/json');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url); //发贴地址
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_setopt($ch, CURLOPT_POSTFIELDS, $params);
$response = curl_exec($ch);//获得返回值
curl_close($ch);// 关闭 cURL 释放资源
if ($error = curl_error($ch)) {
$result = array(
'status' => false,
'message' => $error,
);
} else {
$result = array(
'status' => true,
'message' => 'ok',
'resp' => json_decode($response, true),
);
}
return $result;
}
docker-compose.yml配置
version: '2.2'
services:
elastic01:
image: elasticsearch:7.4.0
container_name: elastic01
environment:
- node.name=elastic01
- cluster.name=es-docker-cluster
- discovery.seed_hosts=elastic02,elastic03
- cluster.initial_master_nodes=elastic01,elastic02,elastic03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ./elastic01/data:/usr/share/elasticsearch/data
- ./elastic01/logs:/usr/share/elasticsearch/logs
- ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9201:9200
networks:
- elastic
elastic02:
image: elasticsearch:7.4.0
container_name: elastic02
environment:
- node.name=elastic02
- cluster.name=es-docker-cluster
- discovery.seed_hosts=elastic01,elastic03
- cluster.initial_master_nodes=elastic01,elastic02,elastic03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ./elastic02/data:/usr/share/elasticsearch/data
- ./elastic02/logs:/usr/share/elasticsearch/logs
- ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9202:9200
networks:
- elastic
elastic03:
image: elasticsearch:7.4.0
container_name: elastic03
environment:
- node.name=elastic03
- cluster.name=es-docker-cluster
- discovery.seed_hosts=elastic01,elastic02
- cluster.initial_master_nodes=elastic01,elastic02,elastic03
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ./elastic03/data:/usr/share/elasticsearch/data
- ./elastic03/logs:/usr/share/elasticsearch/logs
- ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- ./cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9203:9200
networks:
- elastic
kib01:
depends_on:
- elastic01
image: kibana:7.4.0
container_name: kib01
ports:
- 5601:5601
environment:
ELASTICSEARCH_URL: http://elastic01:9200
ELASTICSEARCH_HOSTS: http://elastic01:9200
I18N_LOCALE: zh-CN
volumes:
- ./kibana.yml:/usr/share/kibana/config/kibana.yml
networks:
- elastic
networks:
elastic:
driver: bridge
kibana.yml配置文件
server.host: "0.0.0.0"
elasticsearch.username: "kibana"
elasticsearch.password: "8cidPDyFc1K6kHyRyFMq"
elasticsearch.yml 配置文件
network.host: 0.0.0.0
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/elastic-certificates.p12