手动搭建Kubernetes1.8高可用集群（4）Master
一、准备
1、etcd集群
2、Node1,Node2上搭建Master,以下所有操作都在Node1和2上进行
3、创建目录，并分发证书
/etc/kubernetes/manifests 属主kube 属组kube-cert 权限0700 /etc/kubernetes/ssl
二、安装kubelet,kubectl
1、复制二进制文件
docker run --rm -v /usr/local/bin:/systembindir quay.io/coreos/hyperkube:v1.8.3_coreos.0 /bin/cp /hyperkube /systembindir/kubectl cp /usr/local/bin/kubectl /usr/local/bin/kubelet
2、安装bash completion
yum install bash-completion -y source /usr/share/bash-completion/bash_completion source <(kubectl completion bash)
三、准备kubelet配置文件
1、/etc/systemd/system/kubelet.service
[Unit] Description=Kubernetes Kubelet Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=docker.service Wants=docker.socket [Service] EnvironmentFile=-/etc/kubernetes/kubelet.env ExecStart=/usr/local/bin/kubelet \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBELET_API_SERVER \ $KUBELET_ADDRESS \ $KUBELET_PORT \ $KUBELET_HOSTNAME \ $KUBE_ALLOW_PRIV \ $KUBELET_ARGS \ $DOCKER_SOCKET \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER Restart=always RestartSec=10s [Install] WantedBy=multi-user.target
2、/etc/kubernetes/kubelet.env
# logging to stderr means we get it in the systemd journal KUBE_LOGTOSTDERR="--logtostderr=true" KUBE_LOG_LEVEL="--v=2" # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address=192.168.1.121 --node-ip=192.168.1.121" # The port for the info server to serve on # KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname KUBELET_HOSTNAME="--hostname-override=node1" KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests \ --cadvisor-port=0 \ --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ --node-status-update-frequency=10s \ --docker-disable-shared-pid=True \ --client-ca-file=/etc/kubernetes/ssl/ca.pem \ --tls-cert-file=/etc/kubernetes/ssl/node-node1.pem \ --tls-private-key-file=/etc/kubernetes/ssl/node-node1-key.pem \ --anonymous-auth=false \ --cgroup-driver=cgroupfs \ --cgroups-per-qos=True \ --fail-swap-on=False \ --enforce-node-allocatable="" --cluster-dns=10.233.0.3 --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml --require-kubeconfig --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --kube-reserved cpu=200m,memory=512M --node-labels=node-role.kubernetes.io/master=true --feature-gates=Initializers=true,PersistentLocalVolumes=False " KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow-privileged=true" KUBELET_CLOUDPROVIDER="" PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
3、/etc/kubernetes/node-kubeconfig.yaml
apiVersion: v1 kind: Config clusters: - name: local cluster: certificate-authority: /etc/kubernetes/ssl/ca.pem server: https://127.0.0.1:6443 users: - name: kubelet user: client-certificate: /etc/kubernetes/ssl/node-node1.pem client-key: /etc/kubernetes/ssl/node-node1-key.pem contexts: - context: cluster: local user: kubelet name: kubelet-cluster.local current-context: kubelet-cluster.local
4、启动kubelet
systemctl start kubelet && systemctl enable kubelet
[root@node1 ~]# ss -tnl State Recv-Q Send-Q Local Address:Port Peer Address:Port LISTEN 0 128 192.168.1.123:10250 *:* LISTEN 0 128 192.168.1.123:2379 *:* LISTEN 0 128 127.0.0.1:2379 *:* LISTEN 0 128 192.168.1.123:2380 *:* LISTEN 0 128 192.168.1.123:10255 *:* LISTEN 0 128 *:22 *:* LISTEN 0 100 127.0.0.1:25 *:* LISTEN 0 128 127.0.0.1:10248 *:* LISTEN 0 128 :::22 :::* LISTEN 0 100 ::1:25 :::*
四、配置kube-proxy,apiserver,scheduler,controller-manager
1、/etc/kubernetes/kube-proxy-kubeconfig.yaml
apiVersion: v1 kind: Config clusters: - name: local cluster: certificate-authority: /etc/kubernetes/ssl/ca.pem server: https://127.0.0.1:6443 users: - name: kube-proxy user: client-certificate: /etc/kubernetes/ssl/kube-proxy-node1.pem client-key: /etc/kubernetes/ssl/kube-proxy-node1-key.pem contexts: - context: cluster: local user: kube-proxy name: kube-proxy-cluster.local current-context: kube-proxy-cluster.local
2、/etc/kubernetes/manifests/kube-proxy.manifest
apiVersion: v1 kind: Pod metadata: name: kube-proxy namespace: kube-system labels: k8s-app: kube-proxy annotations: kubespray.kube-proxy-cert/serial: "DBA85609D00B0FAF" spec: hostNetwork: true dnsPolicy: ClusterFirst containers: - name: kube-proxy image: quay.io/coreos/hyperkube:v1.8.3_coreos.0 imagePullPolicy: IfNotPresent resources: limits: cpu: 500m memory: 2000M requests: cpu: 150m memory: 64M command: - /hyperkube - proxy - --v=2 - --kubeconfig=/etc/kubernetes/kube-proxy-kubeconfig.yaml - --bind-address=192.168.1.121 - --cluster-cidr=10.233.64.0/18 - --proxy-mode=iptables securityContext: privileged: true volumeMounts: - mountPath: /etc/ssl/certs name: ssl-certs-host readOnly: true - mountPath: "/etc/kubernetes/ssl" name: etc-kube-ssl readOnly: true - mountPath: "/etc/kubernetes/kube-proxy-kubeconfig.yaml" name: kubeconfig readOnly: true - mountPath: /var/run/dbus name: var-run-dbus readOnly: false volumes: - name: ssl-certs-host hostPath: path: /etc/pki/tls - name: etc-kube-ssl hostPath: path: "/etc/kubernetes/ssl" - name: kubeconfig hostPath: path: "/etc/kubernetes/kube-proxy-kubeconfig.yaml" - name: var-run-dbus hostPath: path: /var/run/dbus
3、/etc/kubernetes/manifests/kube-apiserver.manifest
apiVersion: v1 kind: Pod metadata: name: kube-apiserver namespace: kube-system labels: k8s-app: kube-apiserver kubespray: v2 annotations: kubespray.etcd-cert/serial: "E0C25EE5CFA19DC6" kubespray.apiserver-cert/serial: "DBA85609D00B0FA5" spec: hostNetwork: true dnsPolicy: ClusterFirst containers: - name: kube-apiserver image: quay.io/coreos/hyperkube:v1.8.3_coreos.0 imagePullPolicy: IfNotPresent resources: limits: cpu: 800m memory: 2000M requests: cpu: 100m memory: 256M command: - /hyperkube - apiserver - --advertise-address=192.168.1.121 - --etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.126:2379 - --etcd-quorum-read=true - --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem - --etcd-certfile=/etc/ssl/etcd/ssl/node-node1.pem - --etcd-keyfile=/etc/ssl/etcd/ssl/node-node1-key.pem - --insecure-bind-address=127.0.0.1 - --apiserver-count=2 - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,GenericAdmissionWebhook,ResourceQuota - --service-cluster-ip-range=10.233.0.0/18 - --service-node-port-range=30000-32767 - --client-ca-file=/etc/kubernetes/ssl/ca.pem - --profiling=false - --repair-malformed-updates=false - --kubelet-client-certificate=/etc/kubernetes/ssl/node-node1.pem - --kubelet-client-key=/etc/kubernetes/ssl/node-node1-key.pem - --service-account-lookup=true - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem - --proxy-client-cert-file=/etc/kubernetes/ssl/apiserver.pem - --proxy-client-key-file=/etc/kubernetes/ssl/apiserver-key.pem - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem - --secure-port=6443 - --insecure-port=8080 - --storage-backend=etcd3 - --runtime-config=admissionregistration.k8s.io/v1alpha1 - --v=2 - --allow-privileged=true - --anonymous-auth=False - --authorization-mode=Node,RBAC - --feature-gates=Initializers=true,PersistentLocalVolumes=False livenessProbe: httpGet: host: 127.0.0.1 path: /healthz port: 8080 initialDelaySeconds: 30 timeoutSeconds: 10 volumeMounts: - mountPath: /etc/kubernetes name: kubernetes-config readOnly: true - mountPath: /etc/ssl name: ssl-certs-host readOnly: true - mountPath: /etc/pki/tls name: etc-pki-tls readOnly: true - mountPath: /etc/pki/ca-trust name: etc-pki-ca-trust readOnly: true - mountPath: /etc/ssl/etcd/ssl name: etcd-certs readOnly: true volumes: - hostPath: path: /etc/kubernetes name: kubernetes-config - name: ssl-certs-host hostPath: path: /etc/ssl - name: etc-pki-tls hostPath: path: /etc/pki/tls - name: etc-pki-ca-trust hostPath: path: /etc/pki/ca-trust - hostPath: path: /etc/ssl/etcd/ssl name: etcd-certs
4、/etc/kubernetes/kube-scheduler-kubeconfig.yaml
apiVersion: v1 kind: Config clusters: - name: local cluster: certificate-authority: /etc/kubernetes/ssl/ca.pem server: https://127.0.0.1:6443 users: - name: kube-scheduler user: client-certificate: /etc/kubernetes/ssl/kube-scheduler.pem client-key: /etc/kubernetes/ssl/kube-scheduler-key.pem contexts: - context: cluster: local user: kube-scheduler name: kube-scheduler-cluster.local current-context: kube-scheduler-cluster.local
5、/etc/kubernetes/manifests/kube-scheduler.manifest
apiVersion: v1 kind: Pod metadata: name: kube-scheduler namespace: kube-system labels: k8s-app: kube-scheduler annotations: kubespray.scheduler-cert/serial: "DBA85609D00B0FA6" spec: hostNetwork: true dnsPolicy: ClusterFirst containers: - name: kube-scheduler image: quay.io/coreos/hyperkube:v1.8.3_coreos.0 imagePullPolicy: IfNotPresent resources: limits: cpu: 250m memory: 512M requests: cpu: 80m memory: 170M command: - /hyperkube - scheduler - --leader-elect=true - --kubeconfig=/etc/kubernetes/kube-scheduler-kubeconfig.yaml - --profiling=false - --v=2 - --feature-gates=Initializers=true,PersistentLocalVolumes=False livenessProbe: httpGet: host: 127.0.0.1 path: /healthz port: 10251 initialDelaySeconds: 30 timeoutSeconds: 10 volumeMounts: - mountPath: /etc/ssl name: ssl-certs-host readOnly: true - mountPath: /etc/pki/tls name: etc-pki-tls readOnly: true - mountPath: /etc/pki/ca-trust name: etc-pki-ca-trust readOnly: true - mountPath: "/etc/kubernetes/ssl" name: etc-kube-ssl readOnly: true - mountPath: "/etc/kubernetes/kube-scheduler-kubeconfig.yaml" name: kubeconfig readOnly: true volumes: - name: ssl-certs-host hostPath: path: /etc/ssl - name: etc-pki-tls hostPath: path: /etc/pki/tls - name: etc-pki-ca-trust hostPath: path: /etc/pki/ca-trust - name: etc-kube-ssl hostPath: path: "/etc/kubernetes/ssl" - name: kubeconfig hostPath: path: "/etc/kubernetes/kube-scheduler-kubeconfig.yaml"
6、/etc/kubernetes/kube-controller-manager-kubeconfig.yaml
apiVersion: v1 kind: Config clusters: - name: local cluster: certificate-authority: /etc/kubernetes/ssl/ca.pem server: https://127.0.0.1:6443 users: - name: kube-controller-manager user: client-certificate: /etc/kubernetes/ssl/kube-controller-manager.pem client-key: /etc/kubernetes/ssl/kube-controller-manager-key.pem contexts: - context: cluster: local user: kube-controller-manager name: kube-controller-manager-cluster.local current-context: kube-controller-manager-cluster.local
7、/etc/kubernetes/manifests/kube-controller-manager.manifest
apiVersion: v1 kind: Config clusters: - name: local cluster: certificate-authority: /etc/kubernetes/ssl/ca.pem server: https://127.0.0.1:6443 users: - name: kube-controller-manager user: client-certificate: /etc/kubernetes/ssl/kube-controller-manager.pem client-key: /etc/kubernetes/ssl/kube-controller-manager-key.pem contexts: - context: cluster: local user: kube-controller-manager name: kube-controller-manager-cluster.local current-context: kube-controller-manager-cluster.local [root@node1 ~]# cat /etc/kubernetes/manifests/kube-controller-manager.manifest apiVersion: v1 kind: Pod metadata: name: kube-controller-manager namespace: kube-system labels: k8s-app: kube-controller-manager annotations: kubespray.etcd-cert/serial: "E0C25EE5CFA19DC6" kubespray.controller-manager-cert/serial: "DBA85609D00B0FA7" spec: hostNetwork: true dnsPolicy: ClusterFirst containers: - name: kube-controller-manager image: quay.io/coreos/hyperkube:v1.8.3_coreos.0 imagePullPolicy: IfNotPresent resources: limits: cpu: 250m memory: 512M requests: cpu: 100m memory: 100M command: - /hyperkube - controller-manager - --kubeconfig=/etc/kubernetes/kube-controller-manager-kubeconfig.yaml - --leader-elect=true - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem - --root-ca-file=/etc/kubernetes/ssl/ca.pem - --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem - --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem - --enable-hostpath-provisioner=false - --node-monitor-grace-period=40s - --node-monitor-period=5s - --pod-eviction-timeout=5m0s - --profiling=false - --terminated-pod-gc-threshold=12500 - --v=2 - --use-service-account-credentials=true - --feature-gates=Initializers=true,PersistentLocalVolumes=False livenessProbe: httpGet: host: 127.0.0.1 path: /healthz port: 10252 initialDelaySeconds: 30 timeoutSeconds: 10 volumeMounts: - mountPath: /etc/ssl name: ssl-certs-host readOnly: true - mountPath: /etc/pki/tls name: etc-pki-tls readOnly: true - mountPath: /etc/pki/ca-trust name: etc-pki-ca-trust readOnly: true - mountPath: "/etc/kubernetes/ssl" name: etc-kube-ssl readOnly: true - mountPath: "/etc/kubernetes/kube-controller-manager-kubeconfig.yaml" name: kubeconfig readOnly: true volumes: - name: ssl-certs-host hostPath: path: /etc/ssl - name: etc-pki-tls hostPath: path: /etc/pki/tls - name: etc-pki-ca-trust hostPath: path: /etc/pki/ca-trust - name: etc-kube-ssl hostPath: path: "/etc/kubernetes/ssl" - name: kubeconfig hostPath: path: "/etc/kubernetes/kube-controller-manager-kubeconfig.yaml"
四、复制好配置文件Master就完成了
验证！
1、docker ps
[root@node1 ~]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES db4577657f02 bd322856b660 "/hyperkube contro..." 2 hours ago Up 2 hours k8s_kube-controller-manager_kube-controller-manager-node1_kube-system_94f107a782efd1de544b6ff88e6febf6_0 4bcb249c0074 bd322856b660 "/hyperkube schedu..." 2 hours ago Up 2 hours k8s_kube-scheduler_kube-scheduler-node1_kube-system_e9268832d7bb097d50864adb0eb8195c_0 f905e9f0f050 gcr.io/google_containers/pause-amd64:3.0 "/pause" 2 hours ago Up 2 hours k8s_POD_kube-controller-manager-node1_kube-system_94f107a782efd1de544b6ff88e6febf6_0 bc92979a4b0d gcr.io/google_containers/pause-amd64:3.0 "/pause" 2 hours ago Up 2 hours k8s_POD_kube-scheduler-node1_kube-system_e9268832d7bb097d50864adb0eb8195c_0 80d080d3e0c7 bd322856b660 "/hyperkube apiser..." 2 hours ago Up 2 hours k8s_kube-apiserver_kube-apiserver-node1_kube-system_9df14d9952273a82034f526c484e09fc_0 d03a313b02d5 gcr.io/google_containers/pause-amd64:3.0 "/pause" 2 hours ago Up 2 hours k8s_POD_kube-apiserver-node1_kube-system_9df14d9952273a82034f526c484e09fc_0 7f2c7b71d213 bd322856b660 "/hyperkube proxy ..." 2 hours ago Up 2 hours k8s_kube-proxy_kube-proxy-node1_kube-system_e5c42116406a899b73b44f4a1666e444_0 a0c3ddfa4e24 gcr.io/google_containers/pause-amd64:3.0 "/pause" 2 hours ago Up 2 hours k8s_POD_kube-proxy-node1_kube-system_e5c42116406a899b73b44f4a1666e444_0 7c478db0a1ea quay.io/coreos/etcd:v3.2.4 "/usr/local/bin/etcd" 2 hours ago Up 2 hours etcd1
2、kubectl get node
NotReady的原因是kubelet加了网络选项但目前还没有配置网络插件
[root@node1 ~]# kubectl get node NAME STATUS ROLES AGE VERSION node1 NotReady master 2h v1.8.3+coreos.0 node2 NotReady master,node 2h v1.8.3+coreos.0
3、kubectl get po -n kube-system
[root@node1 ~]# kubectl get po -n kube-system NAME READY STATUS RESTARTS AGE kube-apiserver-node1 1/1 Running 0 51m kube-apiserver-node2 1/1 Running 0 52m kube-controller-manager-node1 1/1 Running 0 40m kube-controller-manager-node2 1/1 Running 0 39m kube-proxy-node1 1/1 Running 5 9m kube-scheduler-node1 1/1 Running 0 42m kube-scheduler-node2 1/1 Running 0 42m