k8s etcd-2 Unhealthy HTTP probe failed with statuscode: 503

[root@vlin121 bin]# kubectl get componentstatuses --server=192.168.133.121:8080      k8s验证错误
NAME                 STATUS      MESSAGE                                  ERROR
scheduler            Healthy     ok                                       
controller-manager   Healthy     ok                                       
etcd-2               Unhealthy   HTTP probe failed with statuscode: 503   
etcd-0               Healthy     {"health":"true"}                        
etcd-1               Healthy     {"health":"true"}  

cat /var/log/messge

Aug 25 18:37:21 vlin121 etcd: health check for peer 5e2c50125f89e6fc could not connect: dial tcp 1
92.168.133.122:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")Aug 25 18:37:21 vlin121 etcd: health check for peer ad0d52b54b7da12 could not connect: dial tcp 19
2.168.133.123:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")Aug 25 18:37:21 vlin121 etcd: health check for peer 5e2c50125f89e6fc could not connect: dial tcp 1
92.168.133.122:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")Aug 25 18:37:21 vlin121 etcd: health check for peer ad0d52b54b7da12 could not connect: dial tcp 19
2.168.133.123:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")Aug 25 18:37:22 vlin121 etcd: e8c545a5e7757ebe [logterm: 250, index: 318, vote: 5e2c50125f89e6fc] 
ignored MsgVote from ad0d52b54b7da12 [logterm: 249, index: 14] at term 250: lease is not expired (remaining ticks: 10)

 

[root@vlin121 bin]# etcdctl --endpoints=http://192.168.133.121:2379 cluster-health      验证etcd有个节点出现错误
member ad0d52b54b7da12 is unhealthy: got unhealthy result from http://192.168.133.123:2379
member 5e2c50125f89e6fc is healthy: got healthy result from http://192.168.133.122:2379
member e8c545a5e7757ebe is healthy: got healthy result from http://192.168.133.121:2379

 

[root@vlin121 bin]# scp /etc/etcd/etcd.conf 192.168.133.123:/etc/etcd/

[root@vlin123 docker]# vim /etc/etcd/etcd.conf                上面四处 加主机名 改本机的

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="http://192.168.133.123:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.133.123:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="vlin123.ywz.com"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.133.123:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.133.123:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="vlin121.ywz.com=http://192.168.133.121:2380,vlin122.ywz.com=http://192.168.133.122:238
0,vlin123.ywz.com=http://192.168.133.123:2380"ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

 

 

[root@vlin123 docker]# systemctl restart etcd.service ; systemctl status etcd.service

 

[root@vlin121 bin]# etcdctl --endpoints=http://192.168.133.121:2379 cluster-health
member ad0d52b54b7da12 is healthy: got healthy result from http://192.168.133.123:2379
member 5e2c50125f89e6fc is healthy: got healthy result from http://192.168.133.122:2379
member e8c545a5e7757ebe is healthy: got healthy result from http://192.168.133.121:2379
 

至此 全好了

[root@vlin121 bin]# kubectl get componentstatuses --server=192.168.133.121:8080
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-1               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"}