docker部署ELK集群

docker deploy ELK:
1:Docker 安装 Elasticsearch

docker pull elasticsearch:7.8.0
docker network create elknet
单机部署：docker run -d --name elasticsearch --net host -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.8.0
集群部署：
es.yml：
--------------------------------------------------------------
#集群名称
cluster.name: elasticsearch-cluster
#节点名称
node.name: es-node1
#设置绑定的ip地址，可以使ipv4或者ipv6
#绑定这台机器的任何一个ip
network.bind_host: 0.0.0.0
#设置其他节点和该节点交互的ip地址，如果不设置它会自动判断，值必须是个真是的ip地址
network.publish_host: 172.18.0.2
#设置对外服务的http端口，默认为9200
http.port: 9200
#设置节点之间的tcp端口，默认是9300
transport.tcp.port: 9300
#是否允许跨域REST请求
http.cors.enabled: true
#允许REST请求来自何处
http.cors.allow-origin: "*"
#初始master节点
cluster.initial_master_nodes: es-node1
#节点角色设置
node.master: true 
node.data: true  
#集群的节点列表
discovery.zen.ping.unicast.hosts: ["172.18.0.2","172.18.0.3"]
 
#集群中一直正常运行的，有成为master节点资格的最少节点数，默认为1
#(total number of master-eligible nodes / 2 + 1)
discovery.zen.minimum_master_nodes: 1
--------------------------------------------------------------

在 /etc/sysctl.conf 追加最大虚拟空间限制 vm.max_map_count=655360 使系统配置生效。 
mkdir -p /data/elasticsearch/log/
mkdir -p /data/elasticsearch/data/
chmod -R 777 /data/
chmod -R 777 /opt/es/

 docker run -e ES_JAVA_OPTS="-Xms4g -Xmx4g" -d -p 0.0.0.0:9200:9200 -p 0.0.0.0:9300:9300 -v /opt/es/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v elastic_config:/usr/share/elasticsearch/ -v /data/elasticsearch/data/:/usr/share/elasticsearch/data -v /data/elasticsearch/log/:/usr/share/elasticsearch/logs -v elastic:/tmp --network=esnet --name ES01 elasticsearch:7.8.0

2:Docker 安装 Kibana

docker pull kibana:7.8.0
docker run -d --name kibana --net host -p 5601:5601 kibana:7.8.0


server.basePath: "/kibana"  # 部署二级目录
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://192.168.8.148:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"
monitoring.ui.container.elasticsearch.enabled: true



3:Docker 安装 Logstash

docker pull logstash:7.8.0
mkdir /opt/logstash/ -p
cd /opt/logstash/
vim logstash.yml
add:
path.config: /usr/share/logstash/conf.d/*.conf
path.logs: /var/log/logstash

vim conf.d/test.conf

input {
    beats {
    port => 5044
    codec => "json"
}
}

output {
if [fields][service] == "111nginx" {
  elasticsearch {
  hosts => ["*.*.*.*:9200","*.*.*.*:9200"]
  index => "111nginx_log-%{+YYYY-MM-dd}"
 }
}
if [fields][service] == "112nginx" {
  elasticsearch {
  hosts => ["*.*.*.*:9200","*.*.*.*:9200"]
  index => "112nginx_log-%{+YYYY-MM-dd}"
 }
}

}


docker run -it -d -p 5044:5044 --name logstash --net elknet -v /opt/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml -v /opt/logstash/conf.d/:/usr/share/logstash/conf.d/ logstash:7.8.0


4:Docker 安装 Filebeat

docker pull store/elastic/filebeat:7.8.0
mkdir /opt/filebeat/
cd /opt/filebeat/
vim filebeat.docker.yml
eg:
---------------------------------------------

filebeat.config:
  modules:
    path: /usr/share/filebeat/modules.d/*.yml
    reload.enabled: false

filebeat.inputs:
- type: log
  enabled: true
  paths:
  - /var/log/nginx/*.log
  input_type: log
  fields.document_type: nginx
  fields.service: 106nginx
  tags: ["nginx"]
output.logstash:
  hosts: ['*.*.*.*:5044']


-----------------------------------
  docker run --name filebeat --user=root -d --net elknet --volume="/var/log/nginx/:/var/log/nginx/" --volume="/opt/filebeat/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" --volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" --volume="/var/run/docker.sock:/var/run/docker.sock:ro" store/elastic/filebeat:7.8.0


删除指定索引的数据post:：
https://*.*.*.*/elasticsearch/dgcn-authuserlogin-2020-08-17/_delete_by_query
{
  "query": {
    "match_all": {}
  }
}


安装elasticdump
npm install elasticdump -g
-g表示全局可用，直接在终端输入 elasticdump --version，出现版本信息即表示安装成功，如下
elasticdump --version
  1、将索引中的数据导出到本地
elasticdump  --input=http://localhost:9200/demo --output=D:/ES/date/demo.json
　其中，demo是索引。
　2、将本地数据导入es中
elasticdump  --input=D:/ES/date/demo.json --output=http://localhost:9200/demo1
　3、将es导入另一个es
elasticdump --input=http://ip:9200/demo --output=http://127.0.0.1:9200/demo