Docker(二十一)--Docker k8s--Kubernetes存储--kubernetes监控--kubernetes资源监控
资源监控
- 1. Metrics-Server
-
- 1.0 前期准备
- 1.1 Metrics-Server部署
- 1.2. 部署后还有不少坑需要填
- 2. Dashboard部署(可视化)
-
- 2.1 拉取镜像
- 2.2 配置
- 2.3. 测试:登陆火狐浏览器输入https://172.25.13.101,访问成功，但是需要token，授权
- 2.4 授权
- 2.5 默认dashboard对集群没有操作权限，需要授权
- 2.6 通过web操作集群
- 2.7 补充
1. Metrics-Server
1.0 前期准备
k8s中文官网
gitee官网
1.1 Metrics-Server部署
-
Metrics-Server是集群核心监控数据(cpu、内存)的聚合器，用来替换之前的heapster。
-
容器相关的 Metrics 主要来自于 kubelet 内置的 cAdvisor 服务，有了Metrics-Server之后，用户就可以通过标准的 Kubernetes API 来访问到这些监控数据。
Metrics API 只可以查询当前的度量数据，并不保存历史数据。
Metrics API URI 为 /apis/metrics.k8s.io/，在 k8s.io/metrics 维护。
必须部署 metrics-server 才能使用该 API，metrics-server 通过调用 Kubelet Summary API 获取数据。 -
示例：
http://127.0.0.1:8001/apis/metrics.k8s.io/v1beta1/nodes
http://127.0.0.1:8001/apis/metrics.k8s.io/v1beta1/nodes/
http://127.0.0.1:8001/apis/metrics.k8s.io/v1beta1/namespace//pods/ -
Metrics Server 并不是 kube-apiserver 的一部分，而是通过 Aggregator 这种插件机制，在独立部署的情况下同 kube-apiserver 一起统一对外服务的。
-
kube-aggregator(默认自启动) 其实就是一个根据 URL 选择具体的 API 后端的代理服务器。
-
Metrics-server属于Core metrics(核心指标)，提供API metrics.k8s.io，仅提供Node和Pod的CPU和内存使用情况。而其他Custom Metrics(自定义指标)由Prometheus等组件来完成。
资源下载 -
Metrics-server部署：
$ kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.6/components.yaml
[root@server2 ~]# kubectl top node ##查看是否有metric
error: Metrics API not available
[root@server2 ~]# kubectl api-versions | grep metric
##1.拉取镜像
[root@server1 harbor]# docker pull bitnami/metrics-server:0.4.0 ##拉取镜像，后面可以指定版本，最新的0.4.2
[root@server1 harbor]# docker tag bitnami/metrics-server:0.4.0 reg.westos.org/library/metrics-server:0.4.0 ##修改名字
[root@server1 harbor]# docker push reg.westos.org/library/metrics-server:0.4.0 ##上传镜像
##2. 配置
[root@server2 metric-server]# pwd ##创建实验目录
/root/metric-server
[root@server2 metric-server]# wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml ##部署配置文件
[root@server2 metric-server]# ll components.yaml
[root@server2 metric-server]# vim components.yaml
[root@server2 metric-server]# cat components.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=4443
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
image: metrics-server:0.4.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 10
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 100
[root@server2 metric-server]# kubectl -n kube-system get pod ##运行但是没有准备好
[root@server2 metric-server]# kubectl -n kube-system get svc ##查看服务
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.96.0.10 53/UDP,53/TCP,9153/TCP 11d
metrics-server ClusterIP 10.111.23.49 443/TCP 3m
[root@server2 metric-server]# kubectl api-versions | grep metric
metrics.k8s.io/v1beta1
[root@server2 metric-server]# kubectl -n kube-system describe pod metrics-server-cc476ccf8-sq92s ##没有运行成功查看详细描述
[root@server2 metric-server]# kubectl -n kube-system logs metrics-server-cc476ccf8-sq92s ##运行成功出错查看日志
[root@server2 metric-server]# vim /var/lib/kubelet/config.yaml ##解决错误2。每个机子都需要执行修改配置文件并重启。最后后一行添加serverTLSBootstrap: true
[root@server2 metric-server]# systemctl restart kubelet.service
[root@server2 metric-server]# kubectl get csr ##查看csr
[root@server2 metric-server]# kubectl certificate approve csr-252k4 csr-bxcq2 csr-d7wd9 csr-dbl6s csr-q55sf csr-tfhpk ####签发证书
[root@server2 metric-server]# kubectl -n kube-system get pod ##查看是否running
[root@server2 metric-server]# kubectl -n kube-system top pod ##查看pod分配情况
[root@server2 metric-server]# kubectl top node
2.配置
1.2. 部署后还有不少坑需要填
部署后查看Metrics-server的Pod日志：
错误1：dial tcp: lookup server2 on 10.96.0.10:53: no such host
这是因为没有内网的DNS服务器，所以metrics-server无法解析节点名字。可以直接修改coredns的configmap，讲各个节点的主机名加入到hosts中，这样所有Pod都可以从CoreDNS中解析各个节点的名字。
$ kubectl edit configmap coredns -n kube-system
apiVersion: v1
data:
Corefile: |
...
ready
hosts {
172.25.0.11 server1
172.25.0.12 server2
172.25.0.13 server3
fallthrough
}
kubernetes cluster.local in-addr.arpa ip6.arpa {
报错2：x509: certificate signed by unknown authority(新版本只有错误2)
Metric Server 支持一个参数 --kubelet-insecure-tls，可以跳过这一检查，然而官方也明确说了，这种方式不推荐生产使用。
启用TLS Bootstrap 证书签发
# vim /var/lib/kubelet/config.yaml
...
serverTLSBootstrap: true
# systemctl restart kubelet
$ kubectl get csr
NAME AGE REQUESTOR CONDITION
csr-f29hk 5s system:node:node-standard-2 Pending
csr-n9pvr 3m31s system:node:node-standard-3 Pending
$ kubectl certificate approve csr-n9pvr ##签发证书
报错3: Error from server (ServiceUnavailable): the server is currently unable to handle the request (get nodes.metrics.k8s.io)
如果metrics-server正常启动，没有错误，应该就是网络问题。修改metrics-server的Pod 网络模式：
hostNetwork: true
2. Dashboard部署(可视化)
- Dashboard可以给用户提供一个可视化的 Web 界面来查看当前集群的各种信息。用户可以用 Kubernetes Dashboard 部署容器化的应用、监控应用的状态、执行故障排查任务以及管理 Kubernetes 各种资源。
网址
##1. 拉取镜像
[root@server1 harbor]# docker pull kubernetesui/dashboard:v2.2.0
[root@server1 harbor]# docker tag kubernetesui/dashboard:v2.2.0 reg.westos.org/kubernetesui/kubernetesui/dashboard:v2.2.0
[root@server1 harbor]# docker push reg.westos.org/kubernetesui/kubernetesui/dashboard:v2.2.0
[root@server1 harbor]# docker pull kubernetesui/metrics-scraper:v1.0.6
[root@server1 harbor]# docker tag kubernetesui/metrics-scraper:v1.0.6 reg.westos.org/kubernetesui/metrics-scraper:v1.0.6
[root@server1 harbor]# docker push reg.westos.org/kubernetesui/metrics-scraper:v1.0.6
## 2. 配置
[root@server2 dashboard]# wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.2.0/aio/deploy/recommended.yaml ##下载配置文件，里面文件不需要改
[root@server2 dashboard]# cat recommended.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
ports:
- port: 443
targetPort: 8443
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-key-holder
namespace: kubernetes-dashboard
type: Opaque
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-settings
namespace: kubernetes-dashboard
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
rules:
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
rules:
# Allow Metrics Scraper to get metrics from the Metrics server
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: kubernetesui/dashboard:v2.2.0
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
spec:
containers:
- name: dashboard-metrics-scraper
image: kubernetesui/metrics-scraper:v1.0.6
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}
[root@server2 dashboard]# kubectl apply -f recommended.yaml
[root@server2 dashboard]# kubectl get ns ##使用metallb为了从外部访问，也可以使用nortport，ingress
metallb-system Active 7d1h
[root@server2 dashboard]# kubectl -n kubernetes-dashboard get pod
[root@server2 dashboard]# kubectl -n metallb-system get pod ##查看metallb对应的pod
[root@server2 dashboard]# kubectl -n kubernetes-dashboard edit svc kubernetes-dashboard
##编辑配置文件，设置为LoadBalancer
[root@server2 dashboard]# kubectl -n kubernetes-dashboard get svc ##查看分配的vip
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-metrics-scraper ClusterIP 10.107.103.161 8000/TCP 6m58s
kubernetes-dashboard LoadBalancer 10.105.128.116 172.25.13.101 443:30250/TCP 6m58s
2.1 拉取镜像
2.2 配置
2.3. 测试:登陆火狐浏览器输入https://172.25.13.101,访问成功，但是需要token，授权
2.4 授权
- 登陆dashboard需要认证，需要获取dashboard pod的token：
$ kubectl describe secrets kubernetes-dashboard-token-g2g6g -n kubernetes-dashboard
[root@server2 dashboard]# kubectl -n kubernetes-dashboard get sa
NAME SECRETS AGE
default 1 14m
kubernetes-dashboard 1 14m
[root@server2 dashboard]# kubectl -n kubernetes-dashboard describe sa kubernetes-dashboard
[root@server2 dashboard]# kubectl describe secrets kubernetes-dashboard-token-c7l79 -n kubernetes-dashboard ##查看token具体信息
##去浏览器输入token信息
2.5 默认dashboard对集群没有操作权限，需要授权
[root@server2 dashboard]# vim rbac.yaml
[root@server2 dashboard]# cat rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
[root@server2 dashboard]# kubectl apply -f rbac.yaml
##去浏览器操作
2.6 通过web操作集群
2.7 补充
- Dashboard如果无法展示metrics-server的数据指标，可以通过以下两种方式解决：
修改dashboard-metrics-scraper 的Pod网络模式： hostNetwork: true
通过node或pod的亲和性设置，让dashboard-metrics-scraper和metrics-server的Pod处于同一个node节点。
