Kubernetes CKA认证运维工程师笔记-Kubernetes故障排查

Kubernetes CKA认证运维工程师笔记-Kubernetes故障排查

  • 1. 应用程序故障排查
  • 2. 管理节点与工作节点异常排查
  • 3. Service访问异常排查

1. 应用程序故障排查

# 置顶type类型,是Pod还是deployment等,再接名称
kubectl describe TYPE/NAME
# 查看日志,一个Pod有两个容器的话,要用-c指定容器名
kubectl logs TYPE/NAME [-c CONTAINER]
# 进入容器中进行查看
kubectl exec POD [-c CONTAINER] --COMMAND [args...]
[root@k8s-master ~]# kubectl get pods
NAME                                     READY   STATUS             RESTARTS   AGE
client1                                  1/1     Running            5          2d23h
client2                                  0/1     ImagePullBackOff   4          2d23h
configmap-demo-pod                       0/1     ImagePullBackOff   3          4d10h
my-pod2                                  1/1     Running            11         4d17h
nfs-client-provisioner-58d675cd5-dx7n4   0/1     ImagePullBackOff   6          4d11h
pod-taint                                1/1     Running            9          10d
secret-demo-pod                          1/1     Running            4          4d9h
sh                                       1/1     Running            6          4d10h
test-76846b5956-gftn9                    1/1     Running            2          4d10h
test-76846b5956-r7s9k                    1/1     Running            2          4d10h
test-76846b5956-trpbn                    1/1     Running            2          4d10h
test2-78c4694588-87b9r                   1/1     Running            5          4d12h
web-0                                    1/1     Running            4          4d11h
web-1                                    0/1     ImagePullBackOff   3          4d11h
web-2                                    0/1     ImagePullBackOff   3          4d11h
web-96d5df5c8-vc9kf                      1/1     Running            3          3d
[root@k8s-master ~]# kubectl get pods
NAME                                     READY   STATUS             RESTARTS   AGE
client1                                  1/1     Running            5          2d23h
client2                                  0/1     ImagePullBackOff   4          2d23h
configmap-demo-pod                       0/1     ImagePullBackOff   3          4d10h
my-pod2                                  1/1     Running            11         4d17h
nfs-client-provisioner-58d675cd5-dx7n4   0/1     ImagePullBackOff   6          4d11h
pod-taint                                1/1     Running            9          10d
secret-demo-pod                          1/1     Running            4          4d9h
sh                                       1/1     Running            6          4d10h
test-76846b5956-gftn9                    1/1     Running            2          4d10h
test-76846b5956-r7s9k                    1/1     Running            2          4d10h
test-76846b5956-trpbn                    1/1     Running            2          4d10h
test2-78c4694588-87b9r                   1/1     Running            5          4d12h
web-0                                    1/1     Running            4          4d11h
web-1                                    0/1     ImagePullBackOff   3          4d11h
web-2                                    0/1     ImagePullBackOff   3          4d11h
web-96d5df5c8-vc9kf                      1/1     Running            3          3d
[root@k8s-master ~]# kubectl describe pod web-96d5df5c8-vc9kf
Name:         web-96d5df5c8-vc9kf
Namespace:    default
Priority:     0
Node:         k8s-node2/10.0.0.63
Start Time:   Wed, 22 Dec 2021 22:11:51 +0800
Labels:       app=web
              pod-template-hash=96d5df5c8
Annotations:  cni.projectcalico.org/podIP: 10.244.169.158/32
              cni.projectcalico.org/podIPs: 10.244.169.158/32
Status:       Running
IP:           10.244.169.158
IPs:
  IP:           10.244.169.158
Controlled By:  ReplicaSet/web-96d5df5c8
Containers:
  nginx:
    Container ID:   docker://f3243ba267e377896e3c5de8a2909d9dd12ed3b2a3fbd80b0094711e5a3f8c81
    Image:          nginx
    Image ID:       docker-pullable://nginx@sha256:366e9f1ddebdb844044c2fafd13b75271a9f620819370f8971220c2b330a9254
    Port:           <none>
    Host Port:      <none>
    State:          Running
      Started:      Sat, 25 Dec 2021 22:09:45 +0800
    Last State:     Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Fri, 24 Dec 2021 15:08:58 +0800
      Finished:     Sat, 25 Dec 2021 22:02:39 +0800
    Ready:          True
    Restart Count:  3
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-8grtj (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  default-token-8grtj:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-8grtj
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                 node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type    Reason          Age    From     Message
  ----    ------          ----   ----     -------
  Normal  SandboxChanged  10m    kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal  Pulling         9m38s  kubelet  Pulling image "nginx"
  Normal  SandboxChanged  4m5s   kubelet  Pod sandbox changed, it will be killed and re-created.
  Normal  Pulling         3m53s  kubelet  Pulling image "nginx"
  Normal  Pulled          3m37s  kubelet  Successfully pulled image "nginx" in 16.296403014s
  Normal  Created         3m36s  kubelet  Created container nginx
  Normal  Started         3m36s  kubelet  Started container nginx
[root@k8s-master ~]# kubectl logs web-96d5df5c8-vc9kf
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
2021/12/25 14:09:46 [notice] 1#1: using the "epoll" event method
2021/12/25 14:09:46 [notice] 1#1: nginx/1.21.4
2021/12/25 14:09:46 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) 
2021/12/25 14:09:46 [notice] 1#1: OS: Linux 3.10.0-1160.45.1.el7.x86_64
2021/12/25 14:09:46 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2021/12/25 14:09:46 [notice] 1#1: start worker processes
2021/12/25 14:09:46 [notice] 1#1: start worker process 31
2021/12/25 14:09:46 [notice] 1#1: start worker process 32
[root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@web-96d5df5c8-vc9kf:/# exit
exit
[root@k8s-master ~]# kubectl exec -it web-96d5df5c8-vc9kf -- bash
root@web-96d5df5c8-vc9kf:/# 

2. 管理节点与工作节点异常排查

管理节点组件:

  • kube-apiserver
  • kube-controller-manager
  • kube-scheduler

工作节点组件:

  • kubelet
  • kube-proxy

Kubernetes集群架构图
Kubernetes CKA认证运维工程师笔记-Kubernetes故障排查_第1张图片
需要先区分部署方式:
1、kubeadm
除kubelet外,其他组件均采用静态Pod启动

[root@k8s-master ~]# ls /etc/kubernetes/manifests/
etcd.yaml            kube-controller-manager.yaml
kube-apiserver.yaml  kube-scheduler.yaml
[root@k8s-master ~]# cat /var/lib/kubelet/config.yaml 
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s

[root@k8s-master ~]# kubectl get pods -n kube-system
NAME                                      READY   STATUS    RESTARTS   AGE
calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          30h
calico-node-4pwdc                         1/1     Running   16         33d
calico-node-9r6zd                         1/1     Running   16         33d
calico-node-vqzdj                         1/1     Running   17         33d
client1                                   1/1     Running   5          2d23h
coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
coredns-6d56c8448f-mdl7c                  1/1     Running   2          30h
etcd-k8s-master                           1/1     Running   3          30h
filebeat-5pwh7                            1/1     Running   11         10d
filebeat-pt848                            1/1     Running   11         10d
kube-apiserver-k8s-master                 1/1     Running   3          30h
kube-controller-manager-k8s-master        1/1     Running   3          30h
kube-proxy-87lbj                          1/1     Running   3          30h
kube-proxy-mcdnv                          1/1     Running   2          30h
kube-proxy-mchc9                          1/1     Running   2          30h
kube-scheduler-k8s-master                 1/1     Running   3          30h
metrics-server-84f9866fdf-rz676           1/1     Running   15         4d15h

2、二进制
所有组件均采用systemd管理

常见问题:

  • 网络不通
    • ping 节点
    • telnet 节点端口
  • 启动失败,一般配置文件或者依赖服务
  • 平台不兼容
# kubeadm部署的,直接查看kube-apiserver-k8s-master的Pod日志
[root@k8s-master ~]# kubectl logs kube-apiserver-k8s-master -n kube-system
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I1225 14:12:18.558307       1 server.go:625] external host was not specified, using 10.0.0.61
I1225 14:12:18.558513       1 server.go:163] Version: v1.19.3
I1225 14:12:19.123232       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:19.123294       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:19.124128       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:19.124167       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:19.126549       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.126601       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.139669       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.139693       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.146921       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.146944       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.147383       1 client.go:360] parsed scheme: "passthrough"
I1225 14:12:19.147669       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:12:19.147718       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:12:19.194115       1 master.go:271] Using reconciler: lease
I1225 14:12:19.194533       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.194550       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.221352       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.221377       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.230469       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.230511       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.240139       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.240181       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.255518       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.255555       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.265105       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.265191       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.275038       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.275076       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.285281       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.285336       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.302076       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.302102       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.314415       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.314679       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.327616       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.327671       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.338580       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.338901       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.354401       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.354487       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.363624       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.363651       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.376090       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.376133       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.386480       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.386534       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.394978       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.395030       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.404842       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.404888       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.559645       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.559692       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.576723       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.576767       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.588265       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.588284       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.596125       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.596145       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.608161       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.608212       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.619144       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.619196       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.626852       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.626895       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.644521       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.644550       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.658031       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.658090       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.669971       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.670265       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.692800       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.692836       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.708784       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.708826       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.734898       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.735032       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.755957       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.755982       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.772847       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.772872       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.788862       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.788886       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.803723       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.803754       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.818516       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.818551       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.826818       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.826857       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.837298       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.837339       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.844194       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.844217       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.857209       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.857597       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.867066       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.867181       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.877262       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.877302       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.889062       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.889099       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.896457       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.902303       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.910393       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.910423       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.927814       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.927861       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.940076       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.940098       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.952012       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.952115       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.961099       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.961123       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.975537       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.975585       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.988067       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.988145       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:19.995939       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:19.995965       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.018436       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.018502       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.109379       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.109398       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.121750       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.121777       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.138751       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.138786       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.148112       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.151713       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.161554       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.161578       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.175335       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.175359       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.193425       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.194080       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.262691       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.262740       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.277204       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.277249       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.299607       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.299713       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.315284       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.315481       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.328823       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.328848       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.345828       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.345871       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.361304       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.361328       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
W1225 14:12:20.640827       1 genericapiserver.go:412] Skipping API batch/v2alpha1 because it has no resources.
W1225 14:12:20.659984       1 genericapiserver.go:412] Skipping API discovery.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.685600       1 genericapiserver.go:412] Skipping API node.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.717635       1 genericapiserver.go:412] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.722620       1 genericapiserver.go:412] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.746581       1 genericapiserver.go:412] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
W1225 14:12:20.774071       1 genericapiserver.go:412] Skipping API apps/v1beta2 because it has no resources.
W1225 14:12:20.774104       1 genericapiserver.go:412] Skipping API apps/v1beta1 because it has no resources.
I1225 14:12:20.794493       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1225 14:12:20.794524       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I1225 14:12:20.801886       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.801939       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:20.810029       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:20.810055       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:23.548796       1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
I1225 14:12:23.548865       1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
I1225 14:12:23.549360       1 dynamic_serving_content.go:130] Starting serving-cert::/etc/kubernetes/pki/apiserver.crt::/etc/kubernetes/pki/apiserver.key
I1225 14:12:23.549780       1 secure_serving.go:197] Serving securely on [::]:6443
I1225 14:12:23.549835       1 dynamic_serving_content.go:130] Starting aggregator-proxy-cert::/etc/kubernetes/pki/front-proxy-client.crt::/etc/kubernetes/pki/front-proxy-client.key
I1225 14:12:23.549858       1 tlsconfig.go:240] Starting DynamicServingCertificateController
I1225 14:12:23.552336       1 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
I1225 14:12:23.552372       1 shared_informer.go:240] Waiting for caches to sync for cluster_authentication_trust_controller
I1225 14:12:23.553014       1 apiservice_controller.go:97] Starting APIServiceRegistrationController
I1225 14:12:23.553087       1 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
I1225 14:12:23.553110       1 controller.go:83] Starting OpenAPI AggregationController
I1225 14:12:23.553250       1 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
I1225 14:12:23.553295       1 dynamic_cafile_content.go:167] Starting request-header::/etc/kubernetes/pki/front-proxy-ca.crt
I1225 14:12:23.561604       1 available_controller.go:404] Starting AvailableConditionController
I1225 14:12:23.561627       1 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
I1225 14:12:23.561671       1 autoregister_controller.go:141] Starting autoregister controller
I1225 14:12:23.561678       1 cache.go:32] Waiting for caches to sync for autoregister controller
I1225 14:12:23.561791       1 customresource_discovery_controller.go:209] Starting DiscoveryController
E1225 14:12:23.666220       1 controller.go:152] Unable to remove old endpoints from kubernetes service: StorageError: key not found, Code: 1, Key: /registry/masterleases/10.0.0.61, ResourceVersion: 0, AdditionalErrorMsg: 
I1225 14:12:23.954656       1 shared_informer.go:247] Caches are synced for cluster_authentication_trust_controller 
I1225 14:12:23.995142       1 crdregistration_controller.go:111] Starting crd-autoregister controller
I1225 14:12:23.995162       1 shared_informer.go:240] Waiting for caches to sync for crd-autoregister
I1225 14:12:23.995170       1 shared_informer.go:247] Caches are synced for crd-autoregister 
I1225 14:12:23.995261       1 controller.go:86] Starting OpenAPI controller
I1225 14:12:24.019280       1 naming_controller.go:291] Starting NamingConditionController
I1225 14:12:24.019448       1 establishing_controller.go:76] Starting EstablishingController
I1225 14:12:24.019750       1 nonstructuralschema_controller.go:186] Starting NonStructuralSchemaConditionController
I1225 14:12:24.021435       1 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
I1225 14:12:24.021505       1 crd_finalizer.go:266] Starting CRDFinalizer
I1225 14:12:24.084663       1 cache.go:39] Caches are synced for AvailableConditionController controller
I1225 14:12:24.089038       1 cache.go:39] Caches are synced for autoregister controller
I1225 14:12:24.155442       1 cache.go:39] Caches are synced for APIServiceRegistrationController controller
I1225 14:12:24.299909       1 trace.go:205] Trace[1198217794]: "Get" url:/api/v1/namespaces/ingress-nginx/secrets/nginx-ingress-serviceaccount-token-vh69r,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:23.797) (total time: 502ms):
Trace[1198217794]: ---"About to write a response" 502ms (14:12:00.299)
Trace[1198217794]: [502.112729ms] [502.112729ms] END
I1225 14:12:24.318639       1 controller.go:606] quota admission added evaluator for: leases.coordination.k8s.io
I1225 14:12:24.356155       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.356216       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:24.389194       1 trace.go:205] Trace[414373803]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node2,user-agent:kubelet/v1.19.0 (linux/amd64) kubernetes/e199641,client:10.0.0.63 (25-Dec-2021 14:12:23.849) (total time: 539ms):
Trace[414373803]: ---"About to write a response" 539ms (14:12:00.389)
Trace[414373803]: [539.865826ms] [539.865826ms] END
I1225 14:12:24.389582       1 trace.go:205] Trace[346194256]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/k8s-node1,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.761) (total time: 627ms):
Trace[346194256]: ---"About to write a response" 627ms (14:12:00.389)
Trace[346194256]: [627.763742ms] [627.763742ms] END
I1225 14:12:24.393405       1 trace.go:205] Trace[538299640]: "Get" url:/api/v1/namespaces/kube-system/pods/kube-apiserver-k8s-master,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.845) (total time: 547ms):
Trace[538299640]: ---"About to write a response" 510ms (14:12:00.356)
Trace[538299640]: [547.414287ms] [547.414287ms] END
I1225 14:12:24.512082       1 trace.go:205] Trace[82502510]: "Create" url:/api/v1/namespaces/default/events,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.61 (25-Dec-2021 14:12:23.846) (total time: 665ms):
Trace[82502510]: ---"Object stored in database" 665ms (14:12:00.511)
Trace[82502510]: [665.364934ms] [665.364934ms] END
I1225 14:12:24.516643       1 trace.go:205] Trace[1819760371]: "GuaranteedUpdate etcd3" type:*core.Event (25-Dec-2021 14:12:23.818) (total time: 698ms):
Trace[1819760371]: ---"Transaction prepared" 459ms (14:12:00.277)
Trace[1819760371]: ---"Transaction committed" 238ms (14:12:00.516)
Trace[1819760371]: [698.586941ms] [698.586941ms] END
I1225 14:12:24.523401       1 trace.go:205] Trace[1243567460]: "Patch" url:/api/v1/namespaces/default/events/configmap-demo-pod.16c404be8eee341b,user-agent:kubelet/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:10.0.0.62 (25-Dec-2021 14:12:23.795) (total time: 721ms):
Trace[1243567460]: ---"Object stored in database" 696ms (14:12:00.516)
Trace[1243567460]: [721.328661ms] [721.328661ms] END
I1225 14:12:24.629195       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.635269       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:24.851524       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.851565       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:24.925875       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:24.925902       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:24.941178       1 storage_scheduling.go:143] all system priority classes are created successfully or already exist.
I1225 14:12:25.002392       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.002477       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:25.093867       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.093912       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
E1225 14:12:25.245098       1 customresource_handler.go:668] error building openapi models for hostendpoints.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.HostEndpoint.properties.spec.properties.ports.items.<array>.properties.protocol has invalid property: anyOf
I1225 14:12:25.245607       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.245627       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:25.277321       1 controller.go:132] OpenAPI AggregationController: action for item : Nothing (removed from the queue).
E1225 14:12:25.283228       1 customresource_handler.go:668] error building openapi models for felixconfigurations.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.kubeNodePortRanges.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.FelixConfiguration.properties.spec.properties.natPortRange has invalid property: anyOf
I1225 14:12:25.284239       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.284261       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
E1225 14:12:25.329108       1 customresource_handler.go:668] error building openapi models for globalnetworkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.GlobalNetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
I1225 14:12:25.330596       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.330710       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:25.357189       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.357217       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:25.392966       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.392992       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
E1225 14:12:25.438707       1 customresource_handler.go:668] error building openapi models for networkpolicies.crd.projectcalico.org: ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.egress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.destination.properties.ports.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.notProtocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.protocol has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.notPorts.items.<array> has invalid property: anyOf
ERROR $root.definitions.org.projectcalico.crd.v1.NetworkPolicy.properties.spec.properties.ingress.items.<array>.properties.source.properties.ports.items.<array> has invalid property: anyOf
I1225 14:12:25.439540       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.439593       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:25.448117       1 trace.go:205] Trace[1794566532]: "Get" url:/apis/rbac.authorization.k8s.io/v1/clusterroles/system:aggregate-to-edit,user-agent:kube-apiserver/v1.19.3 (linux/amd64) kubernetes/1e11e4a,client:::1 (25-Dec-2021 14:12:24.878) (total time: 569ms):
Trace[1794566532]: ---"About to write a response" 569ms (14:12:00.447)
Trace[1794566532]: [569.28003ms] [569.28003ms] END
I1225 14:12:25.654884       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:25.654910       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:26.441116       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:26.441157       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:12:26.579163       1 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
E1225 14:12:29.106613       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
E1225 14:12:34.125834       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": context deadline exceeded
E1225 14:12:39.126932       1 available_controller.go:437] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1: Get "https://10.103.249.20:443/apis/metrics.k8s.io/v1beta1": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
I1225 14:12:41.508946       1 controller.go:606] quota admission added evaluator for: endpoints
I1225 14:12:54.169433       1 client.go:360] parsed scheme: "passthrough"
I1225 14:12:54.169468       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:12:54.169476       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:12:56.191213       1 controller.go:606] quota admission added evaluator for: endpointslices.discovery.k8s.io
I1225 14:12:56.631645       1 client.go:360] parsed scheme: "endpoint"
I1225 14:12:56.631729       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://127.0.0.1:2379  <nil> 0 <nil>}]
I1225 14:13:36.699461       1 client.go:360] parsed scheme: "passthrough"
I1225 14:13:36.699504       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:13:36.699512       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:14:12.928690       1 client.go:360] parsed scheme: "passthrough"
I1225 14:14:12.928831       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:14:12.928859       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:14:51.609220       1 client.go:360] parsed scheme: "passthrough"
I1225 14:14:51.609377       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:14:51.609409       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:15:30.414981       1 client.go:360] parsed scheme: "passthrough"
I1225 14:15:30.415048       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:15:30.415057       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:16:13.416069       1 client.go:360] parsed scheme: "passthrough"
I1225 14:16:13.416140       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:16:13.416158       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:16:53.202182       1 client.go:360] parsed scheme: "passthrough"
I1225 14:16:53.202277       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:16:53.202288       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:17:27.709485       1 client.go:360] parsed scheme: "passthrough"
I1225 14:17:27.709530       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:17:27.709542       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:18:10.159300       1 client.go:360] parsed scheme: "passthrough"
I1225 14:18:10.159338       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:18:10.159345       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:18:40.716569       1 client.go:360] parsed scheme: "passthrough"
I1225 14:18:40.716701       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:18:40.716722       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:19:24.247113       1 client.go:360] parsed scheme: "passthrough"
I1225 14:19:24.247185       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:19:24.247219       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:20:08.987275       1 client.go:360] parsed scheme: "passthrough"
I1225 14:20:08.987543       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:20:08.987583       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:20:44.858512       1 client.go:360] parsed scheme: "passthrough"
I1225 14:20:44.858557       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:20:44.858569       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:21:21.613762       1 client.go:360] parsed scheme: "passthrough"
I1225 14:21:21.613892       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:21:21.614077       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:21:52.143822       1 client.go:360] parsed scheme: "passthrough"
I1225 14:21:52.143911       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:21:52.143929       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:22:27.359651       1 client.go:360] parsed scheme: "passthrough"
I1225 14:22:27.359762       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:22:27.359787       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:23:11.063713       1 client.go:360] parsed scheme: "passthrough"
I1225 14:23:11.063746       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:23:11.063754       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:23:42.744602       1 client.go:360] parsed scheme: "passthrough"
I1225 14:23:42.744670       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:23:42.744688       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:24:15.053047       1 client.go:360] parsed scheme: "passthrough"
I1225 14:24:15.053141       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:24:15.053167       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:24:57.281040       1 client.go:360] parsed scheme: "passthrough"
I1225 14:24:57.286666       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:24:57.286712       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:25:38.863844       1 client.go:360] parsed scheme: "passthrough"
I1225 14:25:38.863903       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:25:38.863912       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:26:18.572451       1 client.go:360] parsed scheme: "passthrough"
I1225 14:26:18.572482       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:26:18.572489       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:26:53.678319       1 client.go:360] parsed scheme: "passthrough"
I1225 14:26:53.678531       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:26:53.678573       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:27:36.433874       1 client.go:360] parsed scheme: "passthrough"
I1225 14:27:36.434093       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:27:36.434125       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:28:18.084057       1 client.go:360] parsed scheme: "passthrough"
I1225 14:28:18.084239       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:28:18.084255       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:28:50.563060       1 client.go:360] parsed scheme: "passthrough"
I1225 14:28:50.563113       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:28:50.563124       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:29:21.855603       1 client.go:360] parsed scheme: "passthrough"
I1225 14:29:21.855751       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:29:21.856461       1 clientconn.go:948] ClientConn switching balancer to "pick_first"
I1225 14:29:52.347034       1 client.go:360] parsed scheme: "passthrough"
I1225 14:29:52.347112       1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379  <nil> 0 <nil>}] <nil> <nil>}
I1225 14:29:52.347130       1 clientconn.go:948] ClientConn switching balancer to "pick_first"


# 二进制部署的,用journalctl -u kube-apiserver
[root@k8s-master ~]# journalctl -u kube-apiserver
-- No entries --
[root@k8s-master ~]# journalctl -u kubelet
-- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022    1419 server.go:411] Version: v1.19.3
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772    1419 server.go:831] Client rotation is on, will bootstrap in background
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425    1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546    1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376    1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464    1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254    1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278    1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786    1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810    1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815    1419 container_manager_linux.go:316] Creating device plugin manager: true
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255    1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336    1419 client.go:94] Start docker client with request timeout=2m0s
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344    1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to 
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377    1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569    1419 docker_service.go:256] Docker cri networking managed by cni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354    1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426    1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518    1419 remote_runtime.go:59] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540    1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629    1419 remote_image.go:50] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637    1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731    1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635762    1419 kubelet.go:273] Watching apiserver
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678016    1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/config/apiserver.go:46: Failed to watch *v1.Pod: failed to l
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678147    1419 reflector.go:127] k8s.io/kubernetes/pkg/kubelet/kubelet.go:438: Failed to watch *v1.Node: failed to list *v1
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.678221    1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.Service: failed to list *v1
Dec 25 22:12:10 k8s-master kubelet[1419]: E1225 22:12:10.977478    1419 aws_credentials.go:77] while getting AWS credentials NoCredentialProviders: no valid providers in chain. Dep
Dec 25 22:12:10 k8s-master kubelet[1419]: For verbose messaging see aws.Config.CredentialsChainVerboseErrors
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.040815    1419 kuberuntime_manager.go:214] Container runtime docker initialized, version: 20.10.11, apiVersion: 1.41.0
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.042898    1419 server.go:1147] Started kubelet
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.044515    1419 kubelet.go:1218] Image garbage collection failed once. Stats initialization may not have completed yet: fail
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.053448    1419 fs_resource_analyzer.go:64] Starting FS ResourceAnalyzer
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.056752    1419 event.go:273] Unable to write event: 'Post "https://10.0.0.61:6443/api/v1/namespaces/default/events": dial t
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062250    1419 volume_manager.go:265] Starting Kubelet Volume Manager
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.062322    1419 server.go:152] Starting to listen on 0.0.0.0:10250
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.071183    1419 server.go:424] Adding debug handlers to kubelet server.
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.079514    1419 desired_state_of_world_populator.go:139] Desired state populator starts to run
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.088034    1419 topology_manager.go:219] [topologymanager] RemoveContainer - Container ID: db716200328937af6f50e1cd3c23d1391
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.102900    1419 controller.go:136] failed to ensure node lease exists, will retry in 200ms, error: Get "https://10.0.0.61:64
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.103298    1419 reflector.go:127] k8s.io/client-go/informers/factory.go:134: Failed to watch *v1.CSIDriver: failed to list *
Dec 25 22:12:11 k8s-master kubelet[1419]: E1225 22:12:11.179554    1419 kubelet.go:2183] node "k8s-master" not found
Dec 25 22:12:11 k8s-master kubelet[1419]: I1225 22:12:11.186533    1419 client.go:87] parsed scheme: "unix"

[root@k8s-master ~]# journalctl -u kubelet > a.txt
[root@k8s-master ~]# more a.txt 
-- Logs begin at Sat 2021-12-25 22:12:00 CST, end at Sat 2021-12-25 22:40:02 CST. --
Dec 25 22:12:07 k8s-master systemd[1]: Started kubelet: The Kubernetes Node Agent.
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729022    1419 server.go:411] Version: v1.19.3
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.729772    1419 server.go:831] Client rotation is on, will bootstrap in background
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.739425    1419 certificate_store.go:130] Loading cert/key pair from "/var/lib/kubelet/pki/kubelet-client-current.pem".
Dec 25 22:12:08 k8s-master kubelet[1419]: I1225 22:12:08.745546    1419 dynamic_cafile_content.go:167] Starting client-ca-bundle::/etc/kubernetes/pki/ca.crt
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.157376    1419 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.208464    1419 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209254    1419 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.209278    1419 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName:
 SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false No
deAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionT
hresholds:[{Signal:nodefs.available Operator:LessThan Value:{Quantity: Percentage:0.1} GracePeriod:0s MinReclaim:} {Signal:nodefs.inodesFree Operator:LessThan Value:{Quan
tity: Percentage:0.05} GracePeriod:0s MinReclaim:} {Signal:imagefs.available Operator:LessThan Value:{Quantity: Percentage:0.15} GracePeriod:0s MinReclaim:} {Si
gnal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUMa
nagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212786    1419 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212810    1419 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.212815    1419 container_manager_linux.go:316] Creating device plugin manager: true
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222255    1419 client.go:77] Connecting to docker on unix:///var/run/docker.sock
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.222336    1419 client.go:94] Start docker client with request timeout=2m0s
Dec 25 22:12:10 k8s-master kubelet[1419]: W1225 22:12:10.261344    1419 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to 
"hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.261377    1419 docker_service.go:241] Hairpin mode set to "hairpin-veth"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.560569    1419 docker_service.go:256] Docker cri networking managed by cni
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591354    1419 docker_service.go:261] Docker Info: &{ID:LZWZ:7SPV:BJT7:3OAX:HPZJ:2U5R:3D3E:SXVB:A5PX:PJX3:3IHG:OEDN Contain
ers:27 ContainersRunning:0 ContainersPaused:0 ContainersStopped:27 Images:16 Driver:overlay2 DriverStatus:[[Backing Filesystem xfs] [Supports d_type true] [Native Overlay Diff true
] [userxattr false]] SystemStatus:[] Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:[] Log:[awslogs fluentd gcplogs gelf journald json-file
 local logentries splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:true KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:tru
e IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6tables:true Debug:false NFd:25 OomKillDisable:true NGoroutines:34 SystemTime:2021-12-25T22:12:10.561900318+08:00 LoggingDrive
r:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:3.10.0-1160.45.1.el7.x86_64 OperatingSystem:CentOS Linux 7 (Core) OSType:linux Architecture:x86_64 IndexServerAddr
ess:https://index.docker.io/v1/ RegistryConfig:0xc00067cc40 NCPU:2 MemTotal:1907732480 GenericResources:[] DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:k8s-ma
ster Labels:[] ExperimentalBuild:false ServerVersion:20.10.11 ClusterStore: ClusterAdvertise: Runtimes:map[io.containerd.runc.v2:{Path:runc Args:[]} io.containerd.runtime.v1.linux:
{Path:runc Args:[]} runc:{Path:runc Args:[]}] DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:[] Nodes:0 Managers:
0 Cluster: Warnings:[]} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:7b11cfaabd73bb80907dd23182b9347b4245eb5d Expected:7b11cfaabd73bb80907dd
23182b9347b4245eb5d} RuncCommit:{ID:v1.0.2-0-g52b36a2 Expected:v1.0.2-0-g52b36a2} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=seccomp,profile=default] ProductLic
ense: Warnings:[]}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.591426    1419 docker_service.go:274] Setting cgroupDriver to cgroupfs
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634518    1419 remote_runtime.go:59] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.634540    1419 remote_runtime.go:59] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635527    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
l> <nil>}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635572    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635629    1419 remote_image.go:50] parsed scheme: ""
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635637    1419 remote_image.go:50] scheme "" not registered, fallback to default scheme
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635651    1419 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{/var/run/dockershim.sock  <nil> 0 <nil>}] <ni
l> <nil>}
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635657    1419 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 25 22:12:10 k8s-master kubelet[1419]: I1225 22:12:10.635731    1419 kubelet.go:261] Adding pod path: /etc/kubernetes/manifests

[root@k8s-master ~]# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
[root@k8s-master ~]# /usr/bin/kubelet
I1225 22:44:25.980513   28854 server.go:411] Version: v1.19.3
W1225 22:44:25.980849   28854 server.go:553] standalone mode, no API client
W1225 22:44:25.980983   28854 container_manager_linux.go:951] CPUAccounting not enabled for pid: 28854
W1225 22:44:25.980991   28854 container_manager_linux.go:954] MemoryAccounting not enabled for pid: 28854
W1225 22:44:26.082976   28854 nvidia.go:61] NVIDIA GPU metrics will not be available: no NVIDIA devices found
W1225 22:44:26.127990   28854 server.go:468] No api server defined - no events will be sent to API server.
I1225 22:44:26.128031   28854 server.go:640] --cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /
I1225 22:44:26.128393   28854 container_manager_linux.go:276] container manager verified user specified cgroup-root exists: []
I1225 22:44:26.128413   28854 container_manager_linux.go:281] Creating Container Manager object based on Node Config: {RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: ContainerRuntime:docker CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:cgroupfs KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:false NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:nodefs.inodesFree Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.15} GracePeriod:0s MinReclaim:<nil>} {Signal:memory.available Operator:LessThan Value:{Quantity:100Mi Percentage:0} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.1} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] ExperimentalCPUManagerPolicy:none ExperimentalCPUManagerReconcilePeriod:10s ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none}
I1225 22:44:26.128818   28854 topology_manager.go:126] [topologymanager] Creating topology manager with none policy
I1225 22:44:26.128827   28854 container_manager_linux.go:311] [topologymanager] Initializing Topology Manager with none policy
I1225 22:44:26.128833   28854 container_manager_linux.go:316] Creating device plugin manager: true
I1225 22:44:26.129807   28854 client.go:77] Connecting to docker on unix:///var/run/docker.sock
I1225 22:44:26.129830   28854 client.go:94] Start docker client with request timeout=2m0s
W1225 22:44:26.183754   28854 docker_service.go:565] Hairpin mode set to "promiscuous-bridge" but kubenet is not enabled, falling back to "hairpin-veth"
I1225 22:44:26.183783   28854 docker_service.go:241] Hairpin mode set to "hairpin-veth"
I1225 22:44:26.247433   28854 docker_service.go:256] Docker cri networking managed by kubernetes.io/no-op
......

3. Service访问异常排查

Service工作流程图
Kubernetes CKA认证运维工程师笔记-Kubernetes故障排查_第2张图片
Service一般是访问不通,有以下可能性:

  1. Service是否关联Pod?
# 查看标签是否正确,已关联到Pod;是否已创建Pod
[root@k8s-master ~]# kubectl get ep
NAME             ENDPOINTS                                           AGE
fuseim.pri-ifs   <none>                                              4d12h
kubernetes       10.0.0.61:6443                                      33d
my-dep           <none>                                              30d
my-service       10.244.36.119:80,10.244.36.122:80,10.244.36.98:80   24d
nginx            10.244.36.119:80,10.244.36.122:80,10.244.36.98:80   4d11h
[root@k8s-master ~]# kubectl get pods
NAME                                     READY   STATUS    RESTARTS   AGE
client1                                  1/1     Running   5          3d
client2                                  1/1     Running   5          3d
configmap-demo-pod                       1/1     Running   4          4d10h
my-pod2                                  1/1     Running   11         4d17h
nfs-client-provisioner-58d675cd5-dx7n4   1/1     Running   7          4d12h
pod-taint                                1/1     Running   9          10d
secret-demo-pod                          1/1     Running   4          4d9h
sh                                       1/1     Running   6          4d11h
test-76846b5956-gftn9                    1/1     Running   2          4d11h
test-76846b5956-r7s9k                    1/1     Running   2          4d11h
test-76846b5956-trpbn                    1/1     Running   2          4d11h
test2-78c4694588-87b9r                   1/1     Running   5          4d12h
web-0                                    1/1     Running   4          4d11h
web-1                                    1/1     Running   4          4d11h
web-2                                    1/1     Running   4          4d11h
web-96d5df5c8-vc9kf                      1/1     Running   3          3d
  1. Service指定target-port端口是否正常?
# 查看service中的target-port是否是指定的端口
[root@k8s-master ~]# kubectl edit svc nginx
Edit cancelled, no changes made.
# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app":"nginx"},"name":"nginx","namespace":"default"},"spec":{"clusterIP":"None","ports":[{"name":"web","port":80}],"selector":{"app":"nginx"}}}
  creationTimestamp: "2021-12-21T02:56:44Z"
  labels:
    app: nginx
  name: nginx
  namespace: default
  resourceVersion: "2334070"
  selfLink: /api/v1/namespaces/default/services/nginx
  uid: 5f07839a-04e4-4214-bbbe-d69357de10d4
spec:
  clusterIP: None
  ports:
  - name: web
    port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}
  1. Pod正常工作吗?
# 查看Pod的IP地址,然后用curl命令去看是否返回正常的内容
[root@k8s-master ~]# kubectl get pods -o wide
NAME                                     READY   STATUS    RESTARTS   AGE     IP               NODE        NOMINATED NODE   READINESS GATES
client1                                  1/1     Running   5          3d      10.244.36.99     k8s-node1   <none>           <none>
client2                                  1/1     Running   5          3d      10.244.36.92     k8s-node1   <none>           <none>
configmap-demo-pod                       1/1     Running   4          4d10h   10.244.36.101    k8s-node1   <none>           <none>
my-pod2                                  1/1     Running   11         4d17h   10.244.169.130   k8s-node2   <none>           <none>
nfs-client-provisioner-58d675cd5-dx7n4   1/1     Running   7          4d12h   10.244.36.116    k8s-node1   <none>           <none>
pod-taint                                1/1     Running   9          10d     10.244.169.132   k8s-node2   <none>           <none>
secret-demo-pod                          1/1     Running   4          4d9h    10.244.36.118    k8s-node1   <none>           <none>
sh                                       1/1     Running   6          4d11h   10.244.36.114    k8s-node1   <none>           <none>
test-76846b5956-gftn9                    1/1     Running   2          4d11h   10.244.36.111    k8s-node1   <none>           <none>
test-76846b5956-r7s9k                    1/1     Running   2          4d11h   10.244.36.100    k8s-node1   <none>           <none>
test-76846b5956-trpbn                    1/1     Running   2          4d11h   10.244.169.185   k8s-node2   <none>           <none>
test2-78c4694588-87b9r                   1/1     Running   5          4d13h   10.244.36.123    k8s-node1   <none>           <none>
web-0                                    1/1     Running   4          4d12h   10.244.36.122    k8s-node1   <none>           <none>
web-1                                    1/1     Running   4          4d12h   10.244.36.119    k8s-node1   <none>           <none>
web-2                                    1/1     Running   4          4d12h   10.244.36.98     k8s-node1   <none>           <none>
web-96d5df5c8-vc9kf                      1/1     Running   3          3d      10.244.169.158   k8s-node2   <none>           <none>
[root@k8s-master ~]# curl 10.244.169.158
  1. Service是否通过DNS工作?
# 查看coredns组件是否正常工作
[root@k8s-master ~]# kubectl get pods -n kube-system
NAME                                      READY   STATUS    RESTARTS   AGE
calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          31h
calico-node-4pwdc                         1/1     Running   16         33d
calico-node-9r6zd                         1/1     Running   16         33d
calico-node-vqzdj                         1/1     Running   17         33d
client1                                   1/1     Running   5          3d
coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
coredns-6d56c8448f-mdl7c                  1/1     Running   2          31h
etcd-k8s-master                           1/1     Running   3          31h
filebeat-5pwh7                            1/1     Running   11         10d
filebeat-pt848                            1/1     Running   11         10d
kube-apiserver-k8s-master                 1/1     Running   3          31h
kube-controller-manager-k8s-master        1/1     Running   3          31h
kube-proxy-87lbj                          1/1     Running   3          31h
kube-proxy-mcdnv                          1/1     Running   2          31h
kube-proxy-mchc9                          1/1     Running   2          31h
kube-scheduler-k8s-master                 1/1     Running   3          31h
metrics-server-84f9866fdf-rz676           1/1     Running   15         4d16h
  1. kube-proxy正常工作吗?
# kubeadm部署的查看kube-proxy看是否工作正常
[root@k8s-master ~]# kubectl get pods -n kube-system
NAME                                      READY   STATUS    RESTARTS   AGE
calico-kube-controllers-97769f7c7-c4cg5   1/1     Running   3          31h
calico-node-4pwdc                         1/1     Running   16         33d
calico-node-9r6zd                         1/1     Running   16         33d
calico-node-vqzdj                         1/1     Running   17         33d
client1                                   1/1     Running   5          3d
coredns-6d56c8448f-gcgrh                  1/1     Running   16         33d
coredns-6d56c8448f-mdl7c                  1/1     Running   2          31h
etcd-k8s-master                           1/1     Running   3          31h
filebeat-5pwh7                            1/1     Running   11         10d
filebeat-pt848                            1/1     Running   11         10d
kube-apiserver-k8s-master                 1/1     Running   3          31h
kube-controller-manager-k8s-master        1/1     Running   3          31h
kube-proxy-87lbj                          1/1     Running   3          31h
kube-proxy-mcdnv                          1/1     Running   2          31h
kube-proxy-mchc9                          1/1     Running   2          31h
kube-scheduler-k8s-master                 1/1     Running   3          31h
metrics-server-84f9866fdf-rz676           1/1     Running   15         4d16h

# 二进制查看systemd

  1. kube-proxy是否正常写iptables规则?
# 使用iptables-save |grep service名称来查看,对应规则是否被创建
[root@k8s-master ~]# iptables -L -n
Chain INPUT (policy ACCEPT)
target     prot opt source               destination         
cali-INPUT  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Cz_u1IQiXIMmKD4c */
KUBE-FIREWALL  all  --  0.0.0.0/0            0.0.0.0/0           

Chain FORWARD (policy DROP)
target     prot opt source               destination         
cali-FORWARD  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:wUHhoiAYhphO9Mso */
KUBE-FORWARD  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding rules */
DOCKER-USER  all  --  0.0.0.0/0            0.0.0.0/0           
DOCKER-ISOLATION-STAGE-1  all  --  0.0.0.0/0            0.0.0.0/0           
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            ctstate RELATED,ESTABLISHED
DOCKER     all  --  0.0.0.0/0            0.0.0.0/0           
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0           
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0           

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination         
cali-OUTPUT  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:tVnHkvAo15HuiPy0 */
KUBE-FIREWALL  all  --  0.0.0.0/0            0.0.0.0/0           

Chain DOCKER (1 references)
target     prot opt source               destination         

Chain DOCKER-ISOLATION-STAGE-1 (1 references)
target     prot opt source               destination         
DOCKER-ISOLATION-STAGE-2  all  --  0.0.0.0/0            0.0.0.0/0           
RETURN     all  --  0.0.0.0/0            0.0.0.0/0           

Chain DOCKER-ISOLATION-STAGE-2 (1 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0           
RETURN     all  --  0.0.0.0/0            0.0.0.0/0           

Chain DOCKER-USER (1 references)
target     prot opt source               destination         
RETURN     all  --  0.0.0.0/0            0.0.0.0/0           

Chain KUBE-FIREWALL (2 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
DROP       all  -- !127.0.0.0/8          127.0.0.0/8          /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT

Chain KUBE-FORWARD (1 references)
target     prot opt source               destination         
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding rules */ mark match 0x4000/0x4000
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding conntrack pod source rule */ ctstate RELATED,ESTABLISHED
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes forwarding conntrack pod destination rule */ ctstate RELATED,ESTABLISHED

Chain KUBE-KUBELET-CANARY (0 references)
target     prot opt source               destination         

Chain cali-FORWARD (1 references)
target     prot opt source               destination         
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:vjrMJCRpqwy5oRoX */ MARK and 0xfff1ffff
cali-from-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:A_sPAO0mcxbT9mOV */ mark match 0x0/0x10000
cali-from-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:8ZoYfO5HKXWbB3pk */
cali-to-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:jdEuaPBe14V2hutn */
cali-to-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:12bc6HljsMKsmfr- */
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:MH9kMp5aNICL-Olv */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000

Chain cali-INPUT (1 references)
target     prot opt source               destination         
ACCEPT     4    --  0.0.0.0/0            0.0.0.0/0            /* cali:PajejrV4aFdkZojI */ /* Allow IPIP packets from Calico hosts */ match-set cali40all-hosts-net src ADDRTYPE match dst-type LOCAL
DROP       4    --  0.0.0.0/0            0.0.0.0/0            /* cali:_wjq-Yrma8Ly1Svo */ /* Drop IPIP packets from non-Calico hosts */
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:ss8lEMQsXi-s6qYT */ MARK and 0xfffff
cali-forward-check  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:PgIW-V0nEjwPhF_8 */
RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:QMJlDwlS0OjHyfMN */ mark match ! 0x0/0xfff00000
cali-wl-to-host  all  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:nDRe73txrna-aZjG */
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:iX2AYvqGXaVqwkro */ mark match 0x10000/0x10000
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:bhpnxD5IRtBP8KW0 */ MARK and 0xfff0ffff
cali-from-host-endpoint  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:H5_bccAbHV0sooVy */
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:inBL01YlfurT0dbI */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000

Chain cali-OUTPUT (1 references)
target     prot opt source               destination         
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Mq1_rAdXXH3YkrzW */ mark match 0x10000/0x10000
cali-forward-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:5Z67OUUpTOM7Xa1a */ mark match ! 0x0/0xfff00000
RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:M2Wf0OehNdig8MHR */
ACCEPT     4    --  0.0.0.0/0            0.0.0.0/0            /* cali:AJBkLho_0Qd8LNr3 */ /* Allow IPIP packets to other Calico hosts */ match-set cali40all-hosts-net dst ADDRTYPE match src-type LOCAL
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:iz2RWXlXJDUfsLpe */ MARK and 0xfff0ffff
cali-to-host-endpoint  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:hXojbnLundZDgZyw */
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:wankpMDC2Cy1KfBv */ /* Host endpoint policy accepted packet. */ mark match 0x10000/0x10000

Chain cali-forward-check (1 references)
target     prot opt source               destination         
RETURN     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Pbldlb4FaULvpdD8 */ ctstate RELATED,ESTABLISHED
cali-set-endpoint-mark  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:ZD-6UxuUtGW-xtzg */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
cali-set-endpoint-mark  udp  --  0.0.0.0/0            0.0.0.0/0           [goto]  /* cali:CbPfUajQ2bFVnDq4 */ /* To kubernetes NodePort service */ multiport dports 30000:32767 match-set cali40this-host dst
cali-set-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:jmhU0ODogX-Zfe5g */ /* To kubernetes service */ ! match-set cali40this-host dst

Chain cali-forward-endpoint-mark (1 references)
target     prot opt source               destination         
cali-from-endpoint-mark  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:O0SmFDrnm7KggWqW */ mark match ! 0x100000/0xfff00000
cali-to-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:aFl0WFKRxDqj8oA6 */
cali-to-hep-forward  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:AZKVrO3i_8cLai5f */
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:96HaP1sFtb-NYoYA */ MARK and 0xfffff
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:VxO6hyNWz62YEtul */ /* Policy explicitly accepted packet. */ mark match 0x10000/0x10000

Chain cali-from-endpoint-mark (1 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:9dpftzl-pNycbr37 */ /* Unknown interface */

Chain cali-from-hep-forward (1 references)
target     prot opt source               destination         

Chain cali-from-host-endpoint (1 references)
target     prot opt source               destination         

Chain cali-from-wl-dispatch (2 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:zTj6P0TIgYvgz-md */ /* Unknown interface */

Chain cali-set-endpoint-mark (3 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:MN61lcxFj1yWuYBo */ /* Unknown endpoint */
MARK       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:nKOjq8N2yzfmS3jk */ /* Non-Cali endpoint mark */ MARK xset 0x100000/0xfff00000

Chain cali-to-hep-forward (2 references)
target     prot opt source               destination         

Chain cali-to-host-endpoint (1 references)
target     prot opt source               destination         

Chain cali-to-wl-dispatch (2 references)
target     prot opt source               destination         
DROP       all  --  0.0.0.0/0            0.0.0.0/0            /* cali:7KNphB1nNHw80nIO */ /* Unknown interface */

Chain cali-wl-to-host (1 references)
target     prot opt source               destination         
cali-from-wl-dispatch  all  --  0.0.0.0/0            0.0.0.0/0            /* cali:Ee9Sbo10IpVujdIY */
ACCEPT     all  --  0.0.0.0/0            0.0.0.0/0            /* cali:nSZbcOoG1xPONxb8 */ /* Configured DefaultEndpointToHostAction */
[root@k8s-master ~]# 
[root@k8s-master ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.96.0.1       <none>        443/TCP        33d
my-dep       NodePort    10.111.199.51   <none>        80:31734/TCP   30d
my-service   NodePort    10.100.228.0    <none>        80:32433/TCP   24d
nginx        ClusterIP   None            <none>        80/TCP         4d12h
[root@k8s-master ~]# iptables-save |grep nginx
[root@k8s-master ~]# iptables-save |grep my-dep
[root@k8s-master ~]# iptables-save |grep my-service
[root@k8s-master ~]# iptables-save |grep kubernetes
-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
-A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A cali-forward-check -p tcp -m comment --comment "cali:ZD-6UxuUtGW-xtzg" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
-A cali-forward-check -p udp -m comment --comment "cali:CbPfUajQ2bFVnDq4" -m comment --comment "To kubernetes NodePort service" -m multiport --dports 30000:32767 -m set --match-set cali40this-host dst -g cali-set-endpoint-mark
-A cali-forward-check -m comment --comment "cali:jmhU0ODogX-Zfe5g" -m comment --comment "To kubernetes service" -m set ! --match-set cali40this-host dst -j cali-set-endpoint-mark
-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
  1. cni网络插件是否正常工作?
    扁平化、转发、实现不同节点间的通信
    还是去查看IP,然后去curl那个IP地址,看是否返回正常值

小结:

  1. 关闭kube-apiserver,执行不了kubectl命令
  2. 关闭kube-controller-manager,创建deployment,但是deployment不会去创建具体的pod
  3. 关闭kube-scheduler,Pod无法调度
  4. 关闭kubelet,Node显示notready
  5. 关闭kube-proxy,网络不通

你可能感兴趣的:(kubernetes,运维,linux)