docker stack部署elfk

通过docker stack部署elfk。elfk最新版本：7.5.1

swarm集群（一个节点）：

manager     192.168.30.128

mkdir -p /home/elfk/{elasticsearch,logstash,kibana,filebeat}/config

mkdir /home/elfk/logstash/pipeline

cd /home/elfk

tree .

.
├── docker-stack.yml
├── elasticsearch
│   ├── config
│   │   └── elasticsearch.yml
│   └── Dockerfile
├── filebeat
│   ├── config
│   │   └── filebeat.yml
│   └── Dockerfile
├── kibana
│   ├── config
│   │   └── kibana.yml
│   └── Dockerfile
└── logstash
    ├── config
    │   └── logstash.yml
    ├── Dockerfile
    └── pipeline
        └── logstash.conf

9 directories, 10 files

---

elasticsearch

Dockerfile

vim /home/elfk/elasticsearch/Dockerfile

ARG ELK_VERSION=7.5.1

# https://github.com/elastic/elasticsearch-docker
# FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
FROM elasticsearch:${ELK_VERSION}
# Add your elasticsearch plugins setup here
# Example: RUN elasticsearch-plugin install analysis-icu

elasticsearch.yml

vim /home/elfk/elasticsearch/config/elasticsearch.yml

---
## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0

## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
#
xpack.license.self_generated.type: trial                #trial为试用版，一个月期限，可更改为basic版本
xpack.security.enabled: true
xpack.monitoring.collection.enabled: true

http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization,X-Requested-With,Content-Length,Content-Type

cd /home/elfk/elasticsearch

docker build -t elfk_elasticsearch:latest .

---

logstash

Dockerfile

vim /home/elfk/logstash/Dockerfile

ARG ELK_VERSION=7.5.1

# https://github.com/elastic/logstash-docker
# FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
FROM logstash:${ELK_VERSION}

# Add your logstash plugins setup here
# Example: RUN logstash-plugin install logstash-filter-json

RUN logstash-plugin install logstash-filter-multiline \
    && logstash-plugin install logstash-output-zabbix

logstash.yml

vim /home/elfk/logstash/config/logstash.yml

---
## Default Logstash configuration from Logstash base image.
## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
#
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]

## X-Pack security credentials
#
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: changeme
xpack.monitoring.collection.interval: 10s

pipeline

vim /home/elfk/logstash/pipeline/logstash.conf

input {
    tcp {
    	port => 5000
    }
}

#input {
#    beats {
#        port => 5044
#    }
#}

output {
    elasticsearch {
    	hosts => "elasticsearch:9200"
    	user => "elastic"
    	password => "changeme"
    }
}

cd /home/elfk/logstash

docker build -t elfk_logstash:latest .

---

kibana

Dockerfile

vim /home/elfk/kibana/Dockerfile

ARG ELK_VERSION=7.5.1

# https://github.com/elastic/kibana-docker
# FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
FROM kibana:${ELK_VERSION}

# Add your kibana plugins setup here
# Example: RUN kibana-plugin install 

kibana.yml

vim /home/elfk/kibana/config/kibana.yml

---
## Default Kibana configuration from Kibana base image.
## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
#
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true

## X-Pack security credentials
#
elasticsearch.username: elastic
elasticsearch.password: changeme

cd /home/elfk/kibana

docker build -t elfk_kibana:latest .

---

docker-stack.yml

vim /home/elfk/docker-stack.yml

version: '3.7'

services:
  elasticsearch:
    image: elfk_elasticsearch:latest
    ports:
      - "9200:9200"
      - "9300:9300"
    configs:
      - source: elastic_config
        target: /usr/share/elasticsearch/config/elasticsearch.yml
    volumes:
      - type: volume
        source: elasticsearch
        target: /usr/share/elasticsearch/data
    environment:
      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
      ELASTIC_PASSWORD: changeme
      discovery.type: single-node
    networks:
      - elk
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.role == manager

  logstash:
    image: elfk_logstash:latest
    ports:
      - "5000:5000"
      - "9600:9600"
    configs:
      - source: logstash_config
        target: /usr/share/logstash/config/logstash.yml
      - source: logstash_pipeline
        target: /usr/share/logstash/pipeline/logstash.conf
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk
    deploy:
      mode: replicated
      replicas: 1

  kibana:
    image: elfk_kibana:latest
    ports:
      - "5601:5601"
    configs:
      - source: kibana_config
        target: /usr/share/kibana/config/kibana.yml
    networks:
      - elk
    deploy:
      mode: replicated
      replicas: 1

  elastichd:  
    image: containerize/elastichd
    ports:
      - "9800:9800"
    networks:
      - elk
    deploy:
      mode: replicated
      replicas: 1

configs:
  elastic_config:
    file: ./elasticsearch/config/elasticsearch.yml
    
  logstash_config:
    file: ./logstash/config/logstash.yml
    
  logstash_pipeline:
    file: ./logstash/pipeline/logstash.conf
    
  kibana_config:
    file: ./kibana/config/kibana.yml
    
volumes:
  elasticsearch:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /home/elfk/elasticsearch/data
    
networks:
  elk:
    driver: overlay

if [ $(grep 'vm.max_map_count' /etc/sysctl.conf |wc -l) -eq 0 ] ; \
then echo 'vm.max_map_count=655360' >> /etc/sysctl.conf; \
fi

sysctl -p

cd /home/elfk

docker stack deploy elfk --compose-file docker-stack.yml

docker service ls

ID                  NAME                 MODE                REPLICAS            IMAGE                           PORTS
mt8b42mup3xw        elfk_elastichd       replicated          1/1                 containerize/elastichd:latest   *:9800->9800/tcp
p1oyya3oj5zv        elfk_elasticsearch   replicated          1/1                 elfk_elasticsearch:latest       *:9200->9200/tcp, *:9300->9300/tcp
qltg67gyh38o        elfk_kibana          replicated          1/1                 elfk_kibana:latest              *:5601->5601/tcp
902xoqx7vyqk        elfk_logstash        replicated          1/1                 elfk_logstash:latest            *:5040->5040/tcp, *:9600->9600/tcp

netstat -lntp

Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      16001/sshd          
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      6158/master         
tcp6       0      0 :::9200                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::5000                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::9300                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::22                   :::*                    LISTEN      16001/sshd          
tcp6       0      0 ::1:25                  :::*                    LISTEN      6158/master         
tcp6       0      0 :::9600                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::5601                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::9800                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::2377                 :::*                    LISTEN      15120/dockerd       
tcp6       0      0 :::7946                 :::*                    LISTEN      15120/dockerd

kibana页面，账号：elastic，密码：changeme

docker stack部署完成，这里省略日志收集测试过程，有兴趣可自行配置logstash，自定义日志处理，并结合zabbix做错误日志告警。

另外，elastichd是类似elasticsearch-head的可视化工具，用来查看elasticsearch。

无密码连接es：http://host:port；带密码连接es：http://user:password@host:port，如http://elastic:changeme@ip:9200。

---

filebeat

Dockerfile

vim /home/elfk/filebeat/Dockerfile

ARG ELK_VERSION=7.5.1

# https://github.com/elastic/beats-docker
# FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}
FROM elastic/filebeat:${ELK_VERSION}
COPY config/filebeat.yml /usr/share/filebeat/filebeat.yml

filebeat.yml

vim /home/elfk/filebeat/config/filebeat.yml

---
filebeat.inputs:
- type: log
  enabled: false
  paths:
    - /var/log/*.log

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  
output.elasticsearch:
  hosts: ['elasticsearch:9200']
  username: elastic
  password: changeme
 
#output.logstash:
  #hosts: ["localhost:5044"]
  #enabled: true
  #worker: 1
  #compression_level: 3

cd /home/elfk/filebeat

docker build -t elfk_filebeat:latest .

filebeat容器在需要收集日志的机器上运行，将日志传输至elasticsearch或logstash。

docker run -d \
  --name=filebeat \
  --user=root \
  -v /var/lib/docker/containers:/var/lib/docker/containers:ro \
  -v /var/run/docker.sock:/var/run/docker.sock:ro \
  -v /home/logs/:/home/logs/:rw \
  -e -strict.perms=false elfk_filebeat:latest

对于要收集的日志，映射时一定要给rw权限，否则即使宿主机上产生新日志，filebeat容器内也不会同步产生新日志。

---