K8s环境部署nginx、tomcat动静分离，部署elk+filebeat日志分析系统。通过nginx代理连接内外网。通过添加节点标签将nginx及tomcat相关pod定向调度至node1节点，elk相关pod定向调度至node2节点，filebeat的kind类型设置为DaemonSet。
节点：
master：172.28.9.90
Node1：172.28.9.92
Node2：172.28.9.91
Nginx负载：172.28.9.93
Nginx、Tomcat网页文件目录：
Node节点标签：
镜像下载：
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.11.1
docker pull docker.elastic.co/logstash/logstash:7.11.1
docker pull docker.elastic.co/kibana/kibana:7.11.1
docker pull docker.elastic.co/beats/filebeat:7.11.1
docker pull docker.io/nginx:latest
docker pull docker.io/tomcat:latest
配置文件：
nginx.yaml、tomcat.yaml、filebeat.yaml、logstash.yaml、elasticsearch.yaml、kibana.yaml
service及pod运行状态：
Nginx及tomcat网页：
Elk网页及nginx日志：
Yaml文件内容：
nginx.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nginx-log-pv
labels:
name: nginx-log-pv
spec:
capacity:
storage: 5Mi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /var/log/nginx/
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx-log-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Mi
selector:
matchLabels:
name: nginx-log-pv
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cm-nginx-config
data:
nginx.conf: |
user nginx;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
upstream tomcat_server1 {
server tomcat-sts-0.tomcatserver.default:8080;
server tomcat-sts-1.tomcatserver.default:8080;
}
upstream tomcat_server2 {
server tomcat-sts-0.tomcatserver.default:8081;
server tomcat-sts-1.tomcatserver.default:8081;
}
server {
listen 80;
server_name test1;
location / {
root /usr/share/nginx/html1;
index test1.html;
}
location /docs {
alias /usr/share/nginx/docs1/;
index test1-docs.html;
}
location ~ .*.jsp$ {
proxy_pass http://tomcat_server1;
proxy_set_header Host $host:$server_port;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
}
server {
listen 81;
server_name test2;
location / {
root /usr/share/nginx/html2;
index test2.html;
}
location /docs {
alias /usr/share/nginx/docs2/;
index test2-docs.html;
}
location ~ .*.jsp$ {
proxy_pass http://tomcat_server2;
proxy_set_header Host $host:$server_port;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
spec:
selector:
matchLabels:
app: nginx
replicas: 2
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: docker.io/nginx:latest
imagePullPolicy: Never
command: [ "nginx" ]
args:
- "-g daemon off;"
- -c
- /etc/nginx/nginx.conf
ports:
- containerPort: 80
name: test1
- containerPort: 81
name: test2
volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
- name: nginx-html-docs
mountPath: /usr/share/nginx/
- name: nginx-log-pvc
mountPath: /var/log/nginx/
nodeSelector:
type: web
volumes:
- name: nginx-config
configMap:
name: cm-nginx-config
- name: nginx-log-pvc
persistentVolumeClaim:
claimName: nginx-log-pvc
- name: nginx-html-docs
hostPath:
path: /usr/share/nginx/
type: Directory
---
apiVersion: v1
kind: Service
metadata:
name: nginx
spec:
type: NodePort
ports:
- name: nginx1
port: 80
nodePort: 30001
- name: nginx2
port: 81
nodePort: 30002
selector:
app: nginx
tomcat.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: tomcat-log-pv
labels:
name: tomcat-log-pv
spec:
capacity:
storage: 5Mi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /var/log/tomcat/
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tomcat-log-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Mi
selector:
matchLabels:
name: tomcat-log-pv
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cm-tomcat-server-config
data:
server.xml: |
type="org.apache.catalina.UserDatabase" description="User database that can be updated and saved" factory="org.apache.catalina.users.MemoryUserDatabaseFactory" pathname="conf/tomcat-users.xml" /> connectionTimeout="20000" redirectPort="8443" /> resourceName="UserDatabase"/> unpackWARs="true" autoDeploy="true"> prefix="test1_access_log" suffix=".txt" pattern="%{X-Forwarded-For}i %h %l %u %t %r %s %b %{Referer}i %{User-Agent}i" /> connectionTimeout="20000" redirectPort="8443" /> resourceName="UserDatabase"/> unpackWARs="true" autoDeploy="true"> prefix="test2_access_log" suffix=".txt" pattern="%{X-Forwarded-For}i %h %l %u %t %r %s %b %{Referer}i %{User-Agent}i" /> --- apiVersion: apps/v1 kind: StatefulSet metadata: name: tomcat-sts spec: selector: matchLabels: app: tomcat replicas: 2 serviceName: tomcatserver template: metadata: labels: app: tomcat spec: containers: - name: tomcat image: docker.io/tomcat:latest imagePullPolicy: Never command: [ "/bin/sh", "-c" ] args: - "bin/catalina.sh run;" ports: - containerPort: 8080 name: test1 - containerPort: 8081 name: test2 volumeMounts: - name: tomcat-server-config mountPath: /usr/local/tomcat/conf/server.xml subPath: server.xml - name: tomcat-webapp mountPath: /usr/share/tomcat/ - name: tomcat-log-pvc mountPath: /var/log/tomcat/ nodeSelector: type: web volumes: - name: tomcat-server-config configMap: name: cm-tomcat-server-config - name: tomcat-log-pvc persistentVolumeClaim: claimName: tomcat-log-pvc - name: tomcat-webapp hostPath: path: /usr/share/tomcat/ type: Directory --- apiVersion: v1 kind: Service metadata: name: tomcat labels: app: tomcat spec: clusterIP: None ports: - port: 8080 name: test1 - port: 8081 name: test2 selector: app: tomcat filebeat.yaml --- apiVersion: v1 kind: ConfigMap metadata: name: cm-filebeat-config data: filebeat.yml: | filebeat.inputs: - type: log enabled: true paths: - /var/log/nginx/*.log filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false output.logstash: hosts: [ "logstash:5044" ] --- apiVersion: apps/v1 kind: DaemonSet metadata: name: filebeat spec: selector: matchLabels: app: filebeat template: metadata: labels: app: filebeat spec: containers: - name: filebeat image: docker.elastic.co/beats/filebeat:7.11.1 imagePullPolicy: Never command: [ "./filebeat" ] args: - -e - -c - filebeat.yml volumeMounts: - name: filebeat-config mountPath: /usr/share/filebeat/filebeat.yml subPath: filebeat.yml - name: nginx-log-pvc mountPath: /var/log/nginx/ volumes: - name: filebeat-config configMap: name: cm-filebeat-config - name: nginx-log-pvc persistentVolumeClaim: claimName: nginx-log-pvc logstash.yaml --- apiVersion: v1 kind: ConfigMap metadata: name: cm-logstash-yml-config data: logstash.yml: | http.host: "0.0.0.0" xpack.monitoring.elasticsearch.hosts: ["http://elasticsearch:9200"] --- apiVersion: v1 kind: ConfigMap metadata: name: cm-logstash-config data: logstash.conf: | input { beats { port => 5044 } } output { elasticsearch { hosts => "elasticsearch:9200" index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" } } --- apiVersion: apps/v1 kind: Deployment metadata: name: logstash spec: replicas: 1 selector: matchLabels: app: logstash template: metadata: labels: app: logstash spec: containers: - image: docker.elastic.co/logstash/logstash:7.11.1 name: logstash imagePullPolicy: Never command: [ "bin/logstash" ] args: - -f - config/logstash.conf ports: - name: log-es containerPort: 5044 volumeMounts: - name: logstash-yml-config mountPath: /usr/share/logstash/config/logstash.yml subPath: logstash.yml - name: logstash-config mountPath: /usr/share/logstash/config/logstash.conf subPath: logstash.conf nodeSelector: type: elk volumes: - name: logstash-yml-config configMap: name: cm-logstash-yml-config - name: logstash-config configMap: name: cm-logstash-config --- apiVersion: v1 kind: Service metadata: name: logstash spec: type: ClusterIP ports: - name: log-es port: 5044 selector: app: logstash elasticsearch.yaml --- apiVersion: v1 kind: ConfigMap metadata: name: cm-elasticsearch-config data: elasticsearch.yml: | network.host: 0.0.0.0 http.port: 9200 cluster.initial_master_nodes: '${POD_IP}' --- apiVersion: apps/v1 kind: Deployment metadata: name: elasticsearch spec: replicas: 1 selector: matchLabels: app: elasticsearch template: metadata: labels: app: elasticsearch spec: containers: - image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1 name: elasticsearch imagePullPolicy: Never env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP ports: - name: es-kibana containerPort: 9200 volumeMounts: - name: elasticsearch-config mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml nodeSelector: type: elk volumes: - name: elasticsearch-config configMap: name: cm-elasticsearch-config --- apiVersion: v1 kind: Service metadata: name: elasticsearch spec: type: ClusterIP ports: - name: log port: 9200 selector: app: elasticsearch kibana.yaml --- apiVersion: v1 kind: ConfigMap metadata: name: cm-kibana-config data: kibana.yml: | server.port: 5601 server.host: "0.0.0.0" elasticsearch.hosts: ["http://elasticsearch:9200"] --- apiVersion: apps/v1 kind: Deployment metadata: name: kibana spec: replicas: 1 selector: matchLabels: app: kibana template: metadata: labels: app: kibana spec: containers: - image: docker.elastic.co/kibana/kibana:7.11.1 name: kibana imagePullPolicy: Never command: [ "bin/kibana" ] args: - '--allow-root' ports: - name: web containerPort: 5601 volumeMounts: - name: kibana-config mountPath: /usr/share/kibana/config/kibana.yml subPath: kibana.yml nodeSelector: type: elk volumes: - name: kibana-config configMap: name: cm-kibana-config --- apiVersion: v1 kind: Service metadata: name: kibana spec: type: NodePort ports: - name: web port: 5601 nodePort: 30003 selector: app: kibana