k8s部署 elfk 7.x + x-pack
k8s以StatefulSet方式部署elasticsearch集群，其中filebeat以sidecar方式部署。当前最新版本：7.10.1
namespace
mkdir -p /home/k8s/elfk/{elasticsearch-head,elasticsearch,logstash,kibana,filebeat}
cd /home/k8s/elfk
vim namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: log
pv
mount -t nfs -o vers=4,minorversion=0,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport xxx.cn-hangzhou.nas.aliyuncs.com:/ /mnt
mkdir -p /mnt/elfk-data/{elasticsearch-0,elasticsearch-1,elasticsearch-2}
vim alicloud-nas-elfk-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: alicloud-nas-elfk-pv0
labels:
alicloud-pvname: alicloud-nas-elfk-pv
spec:
capacity:
storage: 500Gi
accessModes:
- ReadWriteMany
csi:
driver: nasplugin.csi.alibabacloud.com
volumeHandle: alicloud-nas-elfk-pv0
volumeAttributes:
server: "xxx.cn-hangzhou.nas.aliyuncs.com"
path: "/elfk-data/elasticsearch-0"
mountOptions:
- nolock,tcp,noresvport
- vers=4
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: alicloud-nas-elfk-pv1
labels:
alicloud-pvname: alicloud-nas-elfk-pv
spec:
capacity:
storage: 500Gi
accessModes:
- ReadWriteMany
csi:
driver: nasplugin.csi.alibabacloud.com
volumeHandle: alicloud-nas-elfk-pv1
volumeAttributes:
server: "xxx.cn-hangzhou.nas.aliyuncs.com"
path: "/elfk-data/elasticsearch-1"
mountOptions:
- nolock,tcp,noresvport
- vers=4
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: alicloud-nas-elfk-pv2
labels:
alicloud-pvname: alicloud-nas-elfk-pv
spec:
capacity:
storage: 500Gi
accessModes:
- ReadWriteMany
csi:
driver: nasplugin.csi.alibabacloud.com
volumeHandle: alicloud-nas-elfk-pv2
volumeAttributes:
server: "xxx.cn-hangzhou.nas.aliyuncs.com"
path: "/elfk-data/elasticsearch-2"
mountOptions:
- nolock,tcp,noresvport
- vers=4
elasticsearch-haed
vim elasticsearch-head/head.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: head
namespace: log
spec:
rules:
- host: head.lzxlinux.cn
http:
paths:
- path: /
backend:
serviceName: head
servicePort: 9100
---
apiVersion: v1
kind: Service
metadata:
name: head
namespace: log
labels:
app: head
spec:
selector:
app: head
ports:
- port: 9100
protocol: TCP
targetPort: 9100
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: head
namespace: log
labels:
app: head
spec:
replicas: 1
selector:
matchLabels:
app: head
template:
metadata:
labels:
app: head
spec:
containers:
- name: head
image: mobz/elasticsearch-head:5
resources:
limits:
cpu: 200m
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
ports:
- containerPort: 9100
elasticsearch
vim elasticsearch/elasticsearch.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: elasticsearch
namespace: log
spec:
rules:
- host: elasticsearch.lzxlinux.cn
http:
paths:
- path: /
backend:
serviceName: elasticsearch
servicePort: 9200
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
namespace: log
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
ports:
- name: api
port: 9200
- name: discovery
port: 9300
clusterIP: None
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: elasticsearch
namespace: log
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
initContainers:
- name: fix-permissions
image: busybox
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
containers:
- name: elasticsearch
image: elasticsearch:7.10.1
env:
- name: "cluster.name"
value: "elk"
- name: "node.master"
value: "true"
- name: "node.data"
value: "true"
- name: "http.host"
value: "0.0.0.0"
- name: "network.host"
value: "_eth0_"
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: "bootstrap.memory_lock"
value: "false"
- name: "http.port"
value: "9200"
- name: "transport.tcp.port"
value: "9300"
- name: "discovery.seed_hosts"
value: "elasticsearch"
- name: "cluster.initial_master_nodes"
value: "elasticsearch-0,elasticsearch-1,elasticsearch-2"
- name: "discovery.seed_resolver.timeout"
value: "10s"
- name: "discovery.zen.minimum_master_nodes"
value: "2"
- name: "gateway.recover_after_nodes"
value: "3"
- name: "http.cors.enabled"
value: "true"
- name: "http.cors.allow-origin"
value: "*"
- name: "http.cors.allow-headers"
value: "Authorization,X-Requested-With,Content-Length,Content-Type"
- name: "xpack.security.enabled"
value: "true"
- name: "xpack.security.transport.ssl.enabled"
value: "true"
- name: "xpack.security.transport.ssl.verification_mode"
value: "certificate"
- name: "xpack.security.transport.ssl.keystore.path"
value: "elastic-certificates.p12"
- name: "xpack.security.transport.ssl.truststore.path"
value: "elastic-certificates.p12"
- name: "ES_JAVA_OPTS"
value: "-Xms512m -Xmx512m"
ports:
- containerPort: 9200
name: api
protocol: TCP
- containerPort: 9300
name: discovery
protocol: TCP
resources:
limits:
cpu: 1000m
requests:
cpu: 500m
volumeMounts:
- name: cert
mountPath: /usr/share/elasticsearch/config/elastic-certificates.p12
subPath: elastic-certificates.p12
readOnly: true
- name: data
mountPath: /usr/share/elasticsearch/data
terminationGracePeriodSeconds: 30
volumes:
- name: cert
configMap:
name: elastic-certificates
volumeClaimTemplates:
- metadata:
name: data
namespace: log
spec:
selector:
matchLabels:
alicloud-pvname: alicloud-nas-elfk-pv
accessModes: [ "ReadWriteMany" ]
resources:
requests:
storage: 500Gi
kibana
vim kibana/config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: kibana-config
namespace: log
data:
kibana.yml: |
server.port: 5601
server.host: "0"
kibana.index: ".kibana"
elasticsearch.hosts: ["http://elasticsearch:9200"]
elasticsearch.username: kibana_system
elasticsearch.password: elk-2021
i18n.locale: "zh-CN"
vim kibana/kibana.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kibana
namespace: log
spec:
rules:
- host: kibana.lzxlinux.cn
http:
paths:
- path: /
backend:
serviceName: kibana
servicePort: 5601
---
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: log
labels:
app: kibana
spec:
selector:
app: kibana
ports:
- port: 5601
protocol: TCP
targetPort: 5601
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: log
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: kibana:7.10.1
ports:
- containerPort: 5601
resources:
limits:
cpu: 500m
memory: 500Mi
requests:
cpu: 500m
memory: 500Mi
volumeMounts:
- name: config
mountPath: /usr/share/kibana/config
volumes:
- name: config
configMap:
name: kibana-config
logstash
vim logstash/config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: log
data:
logstash.yml: |
http.host: "0"
http.port: 9600
path.config: /usr/share/logstash/pipeline
config.reload.automatic: true
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: logstash_system
xpack.monitoring.elasticsearch.password: elk-2021
xpack.monitoring.elasticsearch.hosts: ["http://elasticsearch:9200"]
xpack.monitoring.collection.interval: 10s
logstash.conf: |
input {
beats {
port => 5040
}
}
filter {
if [type] == "nginx_access" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
if [type] == "tomcat_access" {
grok {
match => { "message" => "(?>%{MONTHDAY}-%{MONTH}-%{YEAR} %{TIME}) %{LOGLEVEL:level} \[%{DATA:exception_info}\] %{GREEDYDATA:message}" }
}
}
}
output {
if [type] == "nginx_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
user => "elastic"
password => "elk-2021"
index => "nginx-log"
}
}
if [type] == "tomcat_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
user => "elastic"
password => "elk-2021"
index => "tomcat-log"
}
}
}
vim logstash/logstash.yaml
apiVersion: v1
kind: Service
metadata:
name: logstash
namespace: log
spec:
selector:
app: logstash
ports:
- protocol: TCP
port: 5040
nodePort: 30040
type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: log
spec:
replicas: 3
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
containers:
- name: logstash
image: logstash:7.10.1
ports:
- containerPort: 9600
- containerPort: 5040
resources:
limits:
cpu: 300m
memory: 1000Mi
requests:
cpu: 300m
memory: 500Mi
volumeMounts:
- name: config
mountPath: /usr/share/logstash/config
- name: pipeline
mountPath: /usr/share/logstash/pipeline
volumes:
- name: config
configMap:
name: logstash-config
items:
- key: logstash.yml
path: logstash.yml
- name: pipeline
configMap:
name: logstash-config
items:
- key: logstash.conf
path: logstash.conf
filebeat
以sidecar方式部署filebeat收集nginx和tomcat日志。
vim filebeat/filebeat-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: default
labels:
app: filebeat
data:
filebeat.yml: |-
filebeat.config:
inputs:
path: ${path.config}/inputs.d/*.yml
reload.enabled: false
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.inputs:
- type: log
enabled: true
tail_files: true
backoff: "1s"
paths:
- /nginxlog/*.log
fields:
pod_name: '${pod_name}'
POD_IP: '${POD_IP}'
type: nginx_access
fields_under_root: true
multiline.pattern: '\d+\.\d+\.\d+\.\d+'
multiline.negate: true
multiline.match: after
- type: log
enabled: true
tail_files: true
backoff: "1s"
paths:
- /tomcatlog/*.log
fields:
pod_name: '${pod_name}'
POD_IP: '${POD_IP}'
type: tomcat_access
fields_under_root: true
multiline.pattern: '\d+\-\w+\-\d+ \d+:\d+:\d+\.\d+'
multiline.negate: true
multiline.match: after
output.logstash:
hosts: ["logstash.log:5040"]
enabled: true
worker: 1
compression_level: 3
vim filebeat/filebeat-nginx.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
namespace: default
labels:
app: nginx
spec:
selector:
app: nginx
ports:
- port: 80
nodePort: 30080
type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
namespace: default
spec:
replicas: 1
minReadySeconds: 15
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.17.0
ports:
- containerPort: 80
volumeMounts:
- name: nginx-log
mountPath: /var/log/nginx
- name: filebeat
image: elastic/filebeat:7.10.1
args: [
"-c", "/etc/filebeat/filebeat.yml",
"-e",
]
env:
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: pod_name
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
runAsUser: 0
volumeMounts:
- name: config
mountPath: /etc/filebeat
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: nginx-log
mountPath: /nginxlog
volumes:
- name: config
configMap:
name: filebeat-config
items:
- key: filebeat.yml
path: filebeat.yml
- name: data
emptyDir: {}
- name: nginx-log
emptyDir: {}
vim filebeat/filebeat-tomcat.yaml
apiVersion: v1
kind: Service
metadata:
name: tomcat
namespace: default
labels:
app: tomcat
spec:
selector:
app: tomcat
ports:
- port: 8080
nodePort: 30880
type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tomcat
namespace: default
spec:
replicas: 1
minReadySeconds: 15
selector:
matchLabels:
app: tomcat
template:
metadata:
labels:
app: tomcat
spec:
terminationGracePeriodSeconds: 30
containers:
- name: tomcat
image: tomcat:8.0.51-alpine
ports:
- containerPort: 8080
volumeMounts:
- name: tomcat-log
mountPath: /usr/local/tomcat/logs
- name: filebeat
image: elastic/filebeat:7.10.1
args: [
"-c", "/etc/filebeat/filebeat.yml",
"-e",
]
env:
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: pod_name
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
runAsUser: 0
volumeMounts:
- name: config
mountPath: /etc/filebeat
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: tomcat-log
mountPath: /tomcatlog
volumes:
- name: config
configMap:
name: filebeat-config
items:
- key: filebeat.yml
path: filebeat.yml
- name: data
emptyDir: {}
- name: tomcat-log
emptyDir: {}
部署
tree .
.
├── alicloud-nas-elfk-pv.yaml
├── elasticsearch
│ ├── elastic-certificates.p12
│ └── elasticsearch.yaml
├── elasticsearch-head
│ └── head.yaml
├── filebeat
│ ├── filebeat-config.yaml
│ ├── filebeat-nginx.yaml
│ └── filebeat-tomcat.yaml
├── kibana
│ ├── config.yaml
│ └── kibana.yaml
├── logstash
│ ├── config.yaml
│ └── logstash.yaml
├── namespace.yaml
5 directories, 12 files
kubectl apply -f namespace.yaml
kubectl create configmap -n log elastic-certificates --from-file=elastic-certificates.p12=elasticsearch/elastic-certificates.p12
kubectl apply -f alicloud-nas-elfk-pv.yaml
kubectl apply -f elasticsearch-head/
kubectl apply -f elasticsearch/
kubectl exec -it -n log elasticsearch-0 -- bash
/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive #自定义密码
Enter password for [elastic]: elk-2021
Reenter password for [elastic]: elk-2021
Enter password for [apm_system]: elk-2021
Reenter password for [apm_system]: elk-2021
Enter password for [kibana]: elk-2021
Reenter password for [kibana]: elk-2021
Enter password for [logstash_system]: elk-2021
Reenter password for [logstash_system]: elk-2021
Enter password for [beats_system]: elk-2021
Reenter password for [beats_system]: elk-2021
Enter password for [remote_monitoring_user]: elk-2021
Reenter password for [remote_monitoring_user]: elk-2021
仅第一次建立集群时需要自定义密码，只要es集群的pv所对应的后端存储还在，即使后面重建es集群也无需再次自定义密码，当然想要修改es的密码除外。
kubectl apply -f kibana/
kubectl apply -f logstash/
kubectl apply -f filebeat/
kubectl get all -n log
NAME READY STATUS RESTARTS AGE
pod/elasticsearch-0 1/1 Running 0 6h5m
pod/elasticsearch-1 1/1 Running 0 6h4m
pod/elasticsearch-2 1/1 Running 0 6h4m
pod/head-5c85b8d699-t9d4w 1/1 Running 0 3d8h
pod/kibana-5cfb7767fd-5x9vb 1/1 Running 0 3h45m
pod/logstash-746ddb77cc-bch5j 1/1 Running 0 25m
pod/logstash-746ddb77cc-mzl8c 1/1 Running 0 25m
pod/logstash-746ddb77cc-t9hvp 1/1 Running 0 25m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/elasticsearch ClusterIP None <none> 9200/TCP,9300/TCP 6h5m
service/head ClusterIP 10.103.231.228 <none> 9100/TCP 3d8h
service/kibana ClusterIP 10.102.83.26 <none> 5601/TCP 32h
service/logstash NodePort 10.111.98.56 <none> 5040:30040/TCP 25m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/head 1/1 1 1 3d8h
deployment.apps/kibana 1/1 1 1 32h
deployment.apps/logstash 3/3 3 3 25m
NAME DESIRED CURRENT READY AGE
replicaset.apps/head-5c85b8d699 1 1 1 3d8h
replicaset.apps/kibana-5cfb7767fd 1 1 1 3h45m
replicaset.apps/logstash-746ddb77cc 3 3 3 25m
NAME READY AGE
statefulset.apps/elasticsearch 3/3 6h5m
测试
- 访问head：
任选一个node ip，在本地添加hosts：
192.168.30.130 elasticsearch.lzxlinux.cn
192.168.30.130 head.lzxlinux.cn
192.168.30.130 kibana.lzxlinux.cn
开启x-pack后访问head需要加上账号及密码，
http://head.lzxlinux.cn/?auth_user=elastic&auth_password=elk-2021
直接使用 http://elasticsearch:9200/ 连接es时会报错，使用 http://elasticsearch.lzxlinux.cn/ 连接则正常。
- 访问kibana：
- 收集nginx日志：
访问nginx页面以产生日志：curl ip:30080，
收集nginx日志没有问题。
- 收集tomcat日志：
访问tomcat页面以产生日志：curl ip:30880，
收集tomcat日志没有问题。
- kibana
@timestamp时区问题：
在kibana日志展示时，@timestamp 时区可能会与日志的时间戳不一致，Stack Management → 高级设置 → Timezone for date formatting，从 Browser 改为 UTC 即可。
如果仍无效，可以修改logstash filter 部分：
filter {
grok {
match => [ "message", "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}" ]
}
date {
match => [ "timestamp", "YYYY-MM-DD HH:mm:ss Z" ]
}
}
之后在kibana上 @timestamp 即为当前时区。
另外，filebeat 如果不是通过 k8s 部署的，连接的 logstash 地址可以填公网 ip:30040，而 logstash 的配置文件中的 5040 端口不变。
至此，k8s部署 elfk 7.x + x-pack 完成。已存放至个人github：kubernetes