第十部分:部署 Ingress 与 Dashboard
1,部署 heapster
官方 dashboard 的github https://github.com/kubernetes/dashboard
官方 heapster 的github https://github.com/kubernetes/heapster
下载 heapster 相关 yaml 文件
wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/grafana.yaml
wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/influxdb.yaml
wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/influxdb/heapster.yaml
wget https://raw.githubusercontent.com/kubernetes/heapster/master/deploy/kube-config/rbac/heapster-rbac.yaml
下载 heapster 镜像下载
# 官方镜像
k8s.gcr.io/heapster-grafana-amd64:v4.4.3
k8s.gcr.io/heapster-amd64:v1.5.3
k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
# 个人的镜像
jicki/heapster-grafana-amd64:v4.4.3
jicki/heapster-amd64:v1.5.3
jicki/heapster-influxdb-amd64:v1.3.3
2,# 替换所有yaml 镜像地址
sed -i 's/k8s\.gcr\.io/jicki/g' *.yaml
3,修改 yaml 文件
# heapster.yaml 文件
#### 修改如下部分 #####
因为 kubelet 启用了 https 所以如下配置需要增加 https 端口
- --source=kubernetes:https://kubernetes.default
修改为
- --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
# heapster-rbac.yaml 文件
#### 修改如下部分 #####
将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定,授予它调用 kubelet API 的权限;
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: heapster
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:heapster
subjects:
- kind: ServiceAccount
name: heapster
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: heapster-kubelet-api
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
name: heapster
namespace: kube-system
4,# 导入所有的文件
[root@kubernetes-110 heapster]# kubectl apply -f .
deployment.extensions/monitoring-grafana created
service/monitoring-grafana created
clusterrolebinding.rbac.authorization.k8s.io/heapster created
clusterrolebinding.rbac.authorization.k8s.io/heapster-kubelet-api created
serviceaccount/heapster created
deployment.extensions/heapster created
service/heapster created
deployment.extensions/monitoring-influxdb created
service/monitoring-influxdb created
5,# 查看运行
[root@kubernetes-110 heapster]# kubectl get pods -n kube-system | grep -E 'heapster|monitoring'
heapster-745d7bc8b7-fknpg 1/1 Running 0 12m
monitoring-grafana-59b4f6d8b7-tzrrw 1/1 Running 0 12m
monitoring-influxdb-f6bcc9795-9wlm8 1/1 Running 0 12m
第十一部分,部署 dashboard
1,下载 dashboard 镜像
# 官方镜像
k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3
# 个人的镜像
jicki/kubernetes-dashboard-amd64:v1.8.3
下载 yaml 文件
curl -O https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml
2,导入 yaml
# 替换所有的 images
sed -i 's/k8s\.gcr\.io/jicki/g' *
3,# 导入文件
[root@kubernetes-110 kubernetes-dashboard]# kubectl apply -f kubernetes-dashboard.yaml
secret/kubernetes-dashboard-certs created
serviceaccount/kubernetes-dashboard created
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
deployment.apps/kubernetes-dashboard created
service/kubernetes-dashboard created
4, 查看
[root@kubernetes-110 kubernetes-dashboard]# kubectl get pods,svc -n kube-system
NAME READY STATUS RESTARTS AGE
pod/coredns-55f86bf584-2rz4q 1/1 Running 0 2h
pod/coredns-55f86bf584-mnd2d 1/1 Running 0 2h
pod/heapster-745d7bc8b7-fknpg 1/1 Running 0 21m
pod/kube-dns-autoscaler-66d448df8f-fd69q 1/1 Running 0 1h
pod/kubernetes-dashboard-65666d4586-tlprc 1/1 Running 0 1m
pod/monitoring-grafana-59b4f6d8b7-tzrrw 1/1 Running 0 21m
pod/monitoring-influxdb-f6bcc9795-9wlm8 1/1 Running 0 21m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/heapster ClusterIP 10.254.10.99
service/kube-dns ClusterIP 10.254.0.2
service/kubernetes-dashboard ClusterIP 10.254.46.166
service/monitoring-grafana ClusterIP 10.254.33.120
service/monitoring-influxdb ClusterIP 10.254.41.59
------------------------
第十二部分,部署 Nginx Ingress
Kubernetes 暴露服务的方式目前只有三种:LoadBlancer Service、NodePort Service、Ingress; 什么是 Ingress ? Ingress 就是利用 Nginx Haproxy 等负载均衡工具来暴露 Kubernetes 服务。
官方 Nginx Ingress github: https://github.com/kubernetes/ingress-nginx/
配置 调度 node
# ingress 有多种方式
1. deployment 自由调度 replicas
2. daemonset 全局调度 分配到所有node里
1,# deployment 自由调度过程中,由于我们需要 约束 controller 调度到指定的 node 中,所以需要对 node 进行 label 标签
# 默认如下:
[root@kubernetes-110 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
kubernetes-113 Ready
kubernetes-114 Ready
kubernetes-115 Ready
# 对 113 与 114,115 打上 label
[root@kubernetes-110 ~]# kubectl label nodes kubernetes-113 ingress=proxy
node/kubernetes-113 labeled
[root@kubernetes-110 ~]# kubectl label nodes kubernetes-114 ingress=proxy
node/kubernetes-114 labeled
[root@kubernetes-110 ~]# kubectl label nodes kubernetes-115 ingress=proxy
node/kubernetes-115 labeled
2,# 查看打完的标签
[root@kubernetes-110 ~]# kubectl get nodes --show-labels
NAME STATUS ROLES AGE VERSION LABELS
kubernetes-113 Ready
kubernetes-114 Ready
kubernetes-115 Ready
3,# 下载镜像
# 官方镜像
gcr.io/google_containers/defaultbackend:1.4
quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2
# 国内镜像
jicki/defaultbackend:1.4
jicki/nginx-ingress-controller:0.16.2
4,# 下载 yaml 文件
# 部署 Nginx backend , Nginx backend 用于统一转发 没有的域名 到指定页面。
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/namespace.yaml
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/default-backend.yaml
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/configmap.yaml
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/tcp-services-configmap.yaml
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/udp-services-configmap.yaml
# 部署 Ingress RBAC 认证
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/rbac.yaml
# 部署 Ingress Controller 组件
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/with-rbac.yaml
5,# tcp-service 与 udp-service, 由于 ingress 不支持 tcp 与 udp 的转发,所以这里配置了两个基于 tcp 与 udp 的 service ,通过 --tcp-services-configmap 与 --udp-services-configmap 来配置 tcp 与 udp 的转发服务
# tcp 例子
apiVersion: v1
kind: ConfigMap
metadata:
name: tcp-services
namespace: ingress-nginx
data:
9000: "default/tomcat:8080"
# 以上配置, 转发 tomcat:8080 端口 到 ingress 节点的 9000 端口中
# udp 例子
apiVersion: v1
kind: ConfigMap
metadata:
name: udp-services
namespace: ingress-nginx
data:
53: "kube-system/kube-dns:53"
6,# 替换所有的 images
sed -i 's/gcr\.io\/google_containers/jicki/g' *
sed -i 's/quay\.io\/kubernetes-ingress-controller/jicki/g' *
7,# 上面 对 两个 node 打了 label 所以配置 replicas: 2,根据节点数据,自行设置
# 修改 yaml 文件 增加 rbac 认证 , hostNetwork 还有 nodeSelector, 第二个 spec 下 增加
vi with-rbac.yaml
spec:
replicas: 2
....
spec:
serviceAccountName: nginx-ingress-serviceaccount
hostNetwork: true
nodeSelector:
ingress: proxy
....
# 这里添加一个 other 端口做为后续tcp转发
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
- name: other
containerPort: 8888
8,# 导入 yaml 文件
创建namespace
[root@kubernetes-110 ingress]# kubectl apply -f namespace.yaml
namespace "ingress-nginx" created
[root@kubernetes-110 nginx-ingress]# kubectl apply -f .
configmap/nginx-configuration created
deployment.extensions/default-http-backend created
service/default-http-backend created
serviceaccount/nginx-ingress-serviceaccount created
clusterrole.rbac.authorization.k8s.io/nginx-ingress-clusterrole created
role.rbac.authorization.k8s.io/nginx-ingress-role created
rolebinding.rbac.authorization.k8s.io/nginx-ingress-role-nisa-binding created
clusterrolebinding.rbac.authorization.k8s.io/nginx-ingress-clusterrole-nisa-binding created
configmap/tcp-services created
configmap/udp-services created
deployment.extensions/nginx-ingress-controller created
9,# 查看服务,可以看到这三个个 pods 被分别调度到 113 与 114,115 中
[root@kubernetes-110 nginx-ingress]# kubectl get pods -n ingress-nginx -o wide
NAME READY STATUS RESTARTS AGE IP NODE
default-http-backend-7f5c6fd7bd-6pwm2 1/1 Running 0 2m 10.254.104.6 kubernetes-115
nginx-ingress-controller-7d874679bb-k8mb6 1/1 Running 0 2m 10.180.160.114 kubernetes-114
nginx-ingress-controller-7d874679bb-n5psp 1/1 Running 0 2m 10.180.160.115 kubernetes-115
nginx-ingress-controller-7d874679bb-qsstq 1/1 Running 0 2m 10.180.160.113 kubernetes-113
10,# 查看我们原有的 svc
[root@kubernetes-110 nginx-ingress]# kubectl get pods
NAME READY STATUS RESTARTS AGE
alpine 1/1 Running 0 3d
nginx-dm-fff68d674-vz2k2 1/1 Running 0 3d
nginx-dm-fff68d674-wdp2k 1/1 Running 0 3d
11,# 创建一个 基于 nginx-dm 的 ingress
vi nginx-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: nginx-ingress
spec:
rules:
- host: nginx.nedy.com
http:
paths:
- backend:
serviceName: nginx-svc
servicePort: 80
导入yaml
[root@kubernetes-110 ~]# kubectl apply -f nginx-ingress.yaml
ingress.extensions/nginx-ingress created
[root@kubernetes-110 ~]# kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
nginx-ingress nginx.nedy.com 80 12s
12,# 测试访问(在访问主机要绑定hosts)
[root@kubernetes-110 ingress]# curl nginx.nedy.com
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
Welcome to nginx!
If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.
For online documentation and support please refer to
Commercial support is available at
Thank you for using nginx.
13,# 创建一个基于 dashboard 的 https 的 ingress
# 新版本的 dashboard 默认就是 ssl ,所以这里使用 tcp 代理到 443 端口
# 查看 dashboard svc
[root@kubernetes-110 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
heapster ClusterIP 10.254.15.60
kube-dns ClusterIP 10.254.0.2
kubernetes-dashboard ClusterIP 10.254.55.253
monitoring-grafana ClusterIP 10.254.54.234
monitoring-influxdb ClusterIP 10.254.49.68
创建dashboard-ingress
[root@kubernetes-110 ~]# vim dashboard-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: dashboard-ingress
spec:
rules:
- host: dashboard.nedy.com
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 80
导入yaml
[root@kubernetes-110 ~]# kubectl apply -f dashboard-ingress.yaml
ingress.extensions/dashboard-ingress created
查看ingress
[root@kubernetes-110 ~]# kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
dashboard-ingress dashboard.nedy.com 80 3s
nginx-ingress nginx.nedy.com 80 16h
# 修改 tcp-services-configmap.yaml 文件
vi tcp-services-configmap.yaml
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
data:
8888: "kube-system/kubernetes-dashboard:443"
# 导入文件
[root@kubernetes-110 nginx-ingress]# kubectl apply -f tcp-services-configmap.yaml
configmap/tcp-services configured
# 查看服务
[root@kubernetes-110 nginx-ingress]# kubectl get configmap/tcp-services -n ingress-nginx
NAME DATA AGE
tcp-services 1 13m
[root@kubernetes-110 nginx-ingress]# kubectl describe configmap/tcp-services -n ingress-nginx
Name: tcp-services
Namespace: ingress-nginx
Labels:
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","data":{"8888":"kube-system/kubernetes-dashboard:443"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"tcp-services","namesp...
Data
====
8888:
----
kube-system/kubernetes-dashboard:443
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal CREATE 14m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
Normal CREATE 13m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
Normal CREATE 13m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
Normal UPDATE 1m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
Normal UPDATE 1m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
Normal UPDATE 1m nginx-ingress-controller ConfigMap ingress-nginx/tcp-services
14,# 测试访问(添加hosts解析)
[root@kubernetes-110 dashboard]# curl -I -k https://dashboard.nedy.com:8888
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: no-store
Content-Length: 990
Content-Type: text/html; charset=utf-8
Last-Modified: Tue, 13 Feb 2018 11:17:03 GMT
Date: Wed, 05 Sep 2018 01:40:23 GMT
15,# 配置一个基于域名的 https , ingress
# 创建一个 基于 自身域名的 证书
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout dashboard.nedy.com-key.key -out dashboard.nedy.com.pem -subj "/CN=dashboard.nedy.com"
# 导入 域名的证书 到 集群 的 secret 中
kubectl create secret tls dashboard-secret --namespace=kube-system --cert dashboard.nedy.com.pem --key dashboard.nedy.com-key.key
# 查看 secret
[root@kubernetes-110 dashboard]# kubectl get secret -n kube-system
NAME TYPE DATA AGE
dashboard-secret kubernetes.io/tls 2 1m
# 创建一个 ingress
vi dashboard-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kube-system
annotations:
ingress.kubernetes.io/ssl-passthrough: "true"
nginx.ingress.kubernetes.io/secure-backends: "true"
spec:
tls:
- hosts:
- dashboard.nedy.com
secretName: dashboard-secret
rules:
- host: dashboard.nedy.com
http:
paths:
- path: /
backend:
serviceName: kubernetes-dashboard
servicePort: 443
# 测试访问( 配置hosts解析)
[root@kubernetes-110 kubernetes-dashboard]# curl -I -k https://dashboard.nedy.com
HTTP/1.1 200 OK
Server: nginx/1.15.3
Date: Wed, 05 Sep 2018 02:02:47 GMT
Content-Type: text/html; charset=utf-8
Content-Length: 990
Connection: keep-alive
Vary: Accept-Encoding
Accept-Ranges: bytes
Cache-Control: no-store
Last-Modified: Tue, 13 Feb 2018 11:17:03 GMT
Strict-Transport-Security: max-age=15724800; includeSubDomains
16,# 登录认证
# 首先创建一个 dashboard rbac 超级用户
vi dashboard-admin-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-admin
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
labels:
k8s-app: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard-admin
namespace: kube-system
查看:
kubectl get ingress -n kube-system
17,# 导入文件
[root@kubernetes-110 kubernetes-dashboard]# kubectl apply -f dashboard-admin-rbac.yaml
serviceaccount/kubernetes-dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-admin created
18,# 查看超级用户的 token 名称
[root@kubernetes-110 kubernetes-dashboard]# kubectl -n kube-system get secret | grep kubernetes-dashboard-admin
kubernetes-dashboard-admin-token-qjkxg kubernetes.io/service-account-token 3 29s
19,# 查看 token 部分
[root@kubernetes-110 kubernetes-dashboard]# kubectl describe -n kube-system secret/kubernetes-dashboard-admin-token-qjkxg
Name: kubernetes-dashboard-admin-token-qjkxg
Namespace: kube-system
Labels:
Annotations: kubernetes.io/service-account.name=kubernetes-dashboard-admin
kubernetes.io/service-account.uid=25422fcb-b0b0-11e8-b88d-005056961600
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1363 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbi10b2tlbi1xamt4ZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjI1NDIyZmNiLWIwYjAtMTFlOC1iODhkLTAwNTA1Njk2MTYwMCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTprdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiJ9.fLKjWpSIm_z4oITmq77VEvZuCyaR625wmHfu5gcBGcfdqnQeJQMLjofjDnX8UR8u3uMETZUk_OUJHBrz86kuD9ctCuMWnsWP9xGcifBsZ74py9t9yXkHoVmoLAiw1xCHW8X_qzoOk_Bnaf3VskZCPRa7YafxPO7f-WibjbcpV7ue3u7q7y6lqyKi793hIaVsEZbh-QDOAqPeFqB8cVc1ahMrji6tafVNbhvLJ_s94n__GFRcT-nMkXZ6YENUF5NG6f2CQnu1rt7yg2OrWonJ-XfD-_aBbMWlULjxN_sYU9p3jFQNcasGEtbLp4CefkpzEoxBYjeJiLRbw3XdUqWHuA
20,# 登录 web ui 选择 令牌登录
https://dashboard.nedy.com