手动搭建Kubernetes1.8高可用集群（4）Master

一、准备

1、etcd集群

2、证书

3、Node1,Node2上搭建Master,以下所有操作都在Node1和2上进行

4、创建目录

/etc/kubernetes/manifests    属主kube  权限0700

二、安装kubelet,kubectl

1、复制二进制文件

docker run --rm -v /usr/local/bin:/systembindir gcr.io/google_containers/hyperkube:v1.8.3 /bin/cp /hyperkube /systembindir/kubectl
cp /usr/local/bin/kubectl /usr/local/bin/kubelet

2、安装bash completion

yum install bash-completion -y
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)

三、准备kubelet配置文件，请注意修改IP地址

1、/etc/systemd/system/kubelet.service

[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Wants=docker.socket

[Service]
EnvironmentFile=-/etc/kubernetes/kubelet.env
ExecStart=/usr/local/bin/kubelet \
                $KUBE_LOGTOSTDERR \
                $KUBE_LOG_LEVEL \
                $KUBELET_API_SERVER \
                $KUBELET_ADDRESS \
                $KUBELET_PORT \
                $KUBELET_HOSTNAME \
                $KUBE_ALLOW_PRIV \
                $KUBELET_ARGS \
                $DOCKER_SOCKET \
                $KUBELET_NETWORK_PLUGIN \
                $KUBELET_CLOUDPROVIDER
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

2、/etc/kubernetes/kubelet.env

# logging to stderr means we get it in the systemd journal
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=2"
# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
KUBELET_ADDRESS="--address=192.168.1.121 --node-ip=192.168.1.121"
# The port for the info server to serve on
# KUBELET_PORT="--port=10250"
# You may leave this blank to use the actual hostname
KUBELET_HOSTNAME="--hostname-override=node1"

KUBELET_ARGS="--pod-manifest-path=/etc/kubernetes/manifests \
--cadvisor-port=0 \
--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \
--node-status-update-frequency=10s \
--docker-disable-shared-pid=True \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--tls-cert-file=/etc/kubernetes/ssl/node-node1.pem \
--tls-private-key-file=/etc/kubernetes/ssl/node-node1-key.pem \
--anonymous-auth=false \
--cgroup-driver=cgroupfs \
--cgroups-per-qos=True \
--fail-swap-on=False \
--enforce-node-allocatable=""  --cluster-dns=10.233.0.3 --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml --require-kubeconfig --register-with-taints=node-role.kubernetes.io/master=:NoSchedule --kube-reserved cpu=200m,memory=512M --node-labels=node-role.kubernetes.io/master=true  --feature-gates=Initializers=true,PersistentLocalVolumes=False  "
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
# Should this cluster be allowed to run privileged docker containers
KUBE_ALLOW_PRIV="--allow-privileged=true"
KUBELET_CLOUDPROVIDER=""

PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

3、/etc/kubernetes/node-kubeconfig.yaml

apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
    server: https://127.0.0.1:6443
users:
- name: kubelet
  user:
    client-certificate: /etc/kubernetes/ssl/node-node1.pem
    client-key: /etc/kubernetes/ssl/node-node1-key.pem
contexts:
- context:
    cluster: local
    user: kubelet
  name: kubelet-cluster.local
current-context: kubelet-cluster.local

4、启动kubelet

systemctl start kubelet && systemctl enable kubelet

四、配置kube-proxy,apiserver,scheduler,controller-manager

1、/etc/kubernetes/kube-proxy-kubeconfig.yaml

apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
    server: https://127.0.0.1:6443
users:
- name: kube-proxy
  user:
    client-certificate: /etc/kubernetes/ssl/kube-proxy-node1.pem
    client-key: /etc/kubernetes/ssl/kube-proxy-node1-key.pem
contexts:
- context:
    cluster: local
    user: kube-proxy
  name: kube-proxy-cluster.local
current-context: kube-proxy-cluster.local

2、/etc/kubernetes/manifests/kube-proxy.manifest

apiVersion: v1
kind: Pod
metadata:
  name: kube-proxy
  namespace: kube-system
  labels:
    k8s-app: kube-proxy
spec:
  hostNetwork: true
  dnsPolicy: ClusterFirst
  containers:
  - name: kube-proxy
    image: gcr.io/google_containers/hyperkube:v1.8.3
    imagePullPolicy: IfNotPresent
    resources:
      limits:
        cpu: 500m
        memory: 2000M
      requests:
        cpu: 150m
        memory: 64M
    command:
    - /hyperkube
    - proxy
    - --v=2
    - --kubeconfig=/etc/kubernetes/kube-proxy-kubeconfig.yaml
    - --bind-address=192.168.1.121
    - --cluster-cidr=10.233.64.0/18
    - --proxy-mode=iptables
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ssl-certs-host
      readOnly: true
    - mountPath: "/etc/kubernetes/ssl"
      name: etc-kube-ssl
      readOnly: true
    - mountPath: "/etc/kubernetes/kube-proxy-kubeconfig.yaml"
      name: kubeconfig
      readOnly: true
    - mountPath: /var/run/dbus
      name: var-run-dbus
      readOnly: false
  volumes:
  - name: ssl-certs-host
    hostPath:
      path: /etc/pki/tls
  - name: etc-kube-ssl
    hostPath:
      path: "/etc/kubernetes/ssl"
  - name: kubeconfig
    hostPath:
      path: "/etc/kubernetes/kube-proxy-kubeconfig.yaml"
  - name: var-run-dbus
    hostPath:
      path: /var/run/dbus

3、/etc/kubernetes/manifests/kube-apiserver.manifest

apiVersion: v1
kind: Pod
metadata:
  name: kube-apiserver
  namespace: kube-system
  labels:
    k8s-app: kube-apiserver
    kubespray: v2
spec:
  hostNetwork: true
  dnsPolicy: ClusterFirst
  containers:
  - name: kube-apiserver
    image: gcr.io/google_containers/hyperkube:v1.8.3
    imagePullPolicy: IfNotPresent
    resources:
      limits:
        cpu: 800m
        memory: 2000M
      requests:
        cpu: 100m
        memory: 256M
    command:
    - /hyperkube
    - apiserver
    - --advertise-address=192.168.1.121
    - --etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.126:2379
    - --etcd-quorum-read=true
    - --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem
    - --etcd-certfile=/etc/ssl/etcd/ssl/node-node1.pem
    - --etcd-keyfile=/etc/ssl/etcd/ssl/node-node1-key.pem
    - --insecure-bind-address=127.0.0.1
    - --apiserver-count=2
    - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,GenericAdmissionWebhook,ResourceQuota
    - --service-cluster-ip-range=10.233.0.0/18
    - --service-node-port-range=30000-32767
    - --client-ca-file=/etc/kubernetes/ssl/ca.pem
    - --profiling=false
    - --repair-malformed-updates=false
    - --kubelet-client-certificate=/etc/kubernetes/ssl/node-node1.pem
    - --kubelet-client-key=/etc/kubernetes/ssl/node-node1-key.pem
    - --service-account-lookup=true
    - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
    - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
    - --proxy-client-cert-file=/etc/kubernetes/ssl/apiserver.pem
    - --proxy-client-key-file=/etc/kubernetes/ssl/apiserver-key.pem
    - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem
    - --secure-port=6443
    - --insecure-port=8080
    - --storage-backend=etcd3
    - --runtime-config=admissionregistration.k8s.io/v1alpha1
    - --v=2
    - --allow-privileged=true
    - --anonymous-auth=False
    - --authorization-mode=Node,RBAC
    - --feature-gates=Initializers=true,PersistentLocalVolumes=False
    livenessProbe:
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 8080
      initialDelaySeconds: 30
      timeoutSeconds: 10
    volumeMounts:
    - mountPath: /etc/kubernetes
      name: kubernetes-config
      readOnly: true
    - mountPath: /etc/ssl
      name: ssl-certs-host
      readOnly: true
    - mountPath: /etc/pki/tls
      name: etc-pki-tls
      readOnly: true
    - mountPath: /etc/pki/ca-trust
      name: etc-pki-ca-trust
      readOnly: true
    - mountPath: /etc/ssl/etcd/ssl
      name: etcd-certs
      readOnly: true
  volumes:
  - hostPath:
      path: /etc/kubernetes
    name: kubernetes-config
  - name: ssl-certs-host
    hostPath:
      path: /etc/ssl
  - name: etc-pki-tls
    hostPath:
      path: /etc/pki/tls
  - name: etc-pki-ca-trust
    hostPath:
      path: /etc/pki/ca-trust
  - hostPath:
      path: /etc/ssl/etcd/ssl
    name: etcd-certs

4、/etc/kubernetes/kube-scheduler-kubeconfig.yaml

apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
    server: https://127.0.0.1:6443
users:
- name: kube-scheduler
  user:
    client-certificate: /etc/kubernetes/ssl/kube-scheduler.pem
    client-key: /etc/kubernetes/ssl/kube-scheduler-key.pem
contexts:
- context:
    cluster: local
    user: kube-scheduler
  name: kube-scheduler-cluster.local
current-context: kube-scheduler-cluster.local

5、/etc/kubernetes/manifests/kube-scheduler.manifest

apiVersion: v1
kind: Pod
metadata:
  name: kube-scheduler
  namespace: kube-system
  labels:
    k8s-app: kube-scheduler
spec:
  hostNetwork: true
  dnsPolicy: ClusterFirst
  containers:
  - name: kube-scheduler
    image: gcr.io/google_containers/hyperkube:v1.8.3
    imagePullPolicy: IfNotPresent
    resources:
      limits:
        cpu: 250m
        memory: 512M
      requests:
        cpu: 80m
        memory: 170M
    command:
    - /hyperkube
    - scheduler
    - --leader-elect=true
    - --kubeconfig=/etc/kubernetes/kube-scheduler-kubeconfig.yaml
    - --profiling=false
    - --v=2
    - --feature-gates=Initializers=true,PersistentLocalVolumes=False
    livenessProbe:
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10251
      initialDelaySeconds: 30
      timeoutSeconds: 10
    volumeMounts:
    - mountPath: /etc/ssl
      name: ssl-certs-host
      readOnly: true
    - mountPath: /etc/pki/tls
      name: etc-pki-tls
      readOnly: true
    - mountPath: /etc/pki/ca-trust
      name: etc-pki-ca-trust
      readOnly: true
    - mountPath: "/etc/kubernetes/ssl"
      name: etc-kube-ssl
      readOnly: true
    - mountPath: "/etc/kubernetes/kube-scheduler-kubeconfig.yaml"
      name: kubeconfig
      readOnly: true
  volumes:
  - name: ssl-certs-host
    hostPath:
      path: /etc/ssl
  - name: etc-pki-tls
    hostPath:
      path: /etc/pki/tls
  - name: etc-pki-ca-trust
    hostPath:
      path: /etc/pki/ca-trust
  - name: etc-kube-ssl
    hostPath:
      path: "/etc/kubernetes/ssl"
  - name: kubeconfig
    hostPath:
      path: "/etc/kubernetes/kube-scheduler-kubeconfig.yaml"

6、/etc/kubernetes/kube-controller-manager-kubeconfig.yaml

apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    certificate-authority: /etc/kubernetes/ssl/ca.pem
    server: https://127.0.0.1:6443
users:
- name: kube-controller-manager
  user:
    client-certificate: /etc/kubernetes/ssl/kube-controller-manager.pem
    client-key: /etc/kubernetes/ssl/kube-controller-manager-key.pem
contexts:
- context:
    cluster: local
    user: kube-controller-manager
  name: kube-controller-manager-cluster.local
current-context: kube-controller-manager-cluster.local

7、/etc/kubernetes/manifests/kube-controller-manager.manifest

apiVersion: v1
kind: Pod
metadata:
  name: kube-controller-manager
  namespace: kube-system
  labels:
    k8s-app: kube-controller-manager
spec:
  hostNetwork: true
  dnsPolicy: ClusterFirst
  containers:
  - name: kube-controller-manager
    image: gcr.io/google_containers/hyperkube:v1.8.3
    imagePullPolicy: IfNotPresent
    resources:
      limits:
        cpu: 250m
        memory: 512M
      requests:
        cpu: 100m
        memory: 100M
    command:
    - /hyperkube
    - controller-manager
    - --kubeconfig=/etc/kubernetes/kube-controller-manager-kubeconfig.yaml
    - --leader-elect=true
    - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
    - --root-ca-file=/etc/kubernetes/ssl/ca.pem
    - --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem
    - --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem
    - --enable-hostpath-provisioner=false
    - --node-monitor-grace-period=40s
    - --node-monitor-period=5s
    - --pod-eviction-timeout=5m0s
    - --profiling=false
    - --terminated-pod-gc-threshold=12500
    - --v=2
    - --use-service-account-credentials=true
    - --feature-gates=Initializers=true,PersistentLocalVolumes=False
    livenessProbe:
      httpGet:
        host: 127.0.0.1
        path: /healthz
        port: 10252
      initialDelaySeconds: 30
      timeoutSeconds: 10
    volumeMounts:
    - mountPath: /etc/ssl
      name: ssl-certs-host
      readOnly: true
    - mountPath: /etc/pki/tls
      name: etc-pki-tls
      readOnly: true
    - mountPath: /etc/pki/ca-trust
      name: etc-pki-ca-trust
      readOnly: true
    - mountPath: "/etc/kubernetes/ssl"
      name: etc-kube-ssl
      readOnly: true
    - mountPath: "/etc/kubernetes/kube-controller-manager-kubeconfig.yaml"
      name: kubeconfig
      readOnly: true
  volumes:
  - name: ssl-certs-host
    hostPath:
      path: /etc/ssl
  - name: etc-pki-tls
    hostPath:
      path: /etc/pki/tls
  - name: etc-pki-ca-trust
    hostPath:
      path: /etc/pki/ca-trust
  - name: etc-kube-ssl
    hostPath:
      path: "/etc/kubernetes/ssl"
  - name: kubeconfig
    hostPath:
      path: "/etc/kubernetes/kube-controller-manager-kubeconfig.yaml"

四、复制好配置文件Master就完成了

NotReady是因为还没有网络，calico起来就好了

所有配置文件的属主都是kube

下一步搭建Node