k8s部署（单节点）

部署思路

1.自签ssl证书
2.etcd数据库集群部署
3.node安装docker
4.flannel容器集群网络部署
5.部署master组件
6.部署node组件
-------------------↑本章节↑-------------------
7.部署一个测试示例
8.部署web ui（dashboard）
9.部署集群内部dns解析服务（coredns）

1.实验环境

master：192.168.58.10
node1：192.168.58.40
node2：192.168.58.50

2.etcd集群部署

准备脚本

ls /root/k8s/
cfssl.sh	#证书制作工具安装脚本
etcd-cert.sh	#etcd证书制作脚本
etcd.sh		#配置etcd服务脚本（生成配置文件、启动脚本，并启动服务）

证书制作工具安装脚本
注：若执行etcd证书制作脚本出错，可能是制作工具下载完整性错误，重新下载保证工具正常

cat cfssl.sh
#!/bin/bash
# download cfssl package
#1.cfssl 生成证书工具
#2.cfssljson 通过传入json文件生成证书
#3.cfssl-certinfo 查看证书信息
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

etcd证书制作脚本

cat etcd-cert.sh
#!/bin/bash
# 创建etcd组件证书
#1.创建ca配置文件，ca-config.json是ca证书的配置文件
cat > ca-config.json <<EOF
{
  "signing": {
     "default": {
        "expiry": "87600h"
    },
    "profiles": {
    "www": {
        "expiry": "87600h",
       "usages": [
           "signing",
           "key encipherment",
           "server auth",
           "client auth"
        ]
     }
   }
  }
}
EOF
#2.创建ca证书签名请求，ca-csr.json是ca证书的签名文件
cat > ca-csr.json <<EOF
{
     "CN": "etcd CA",
     "key": {
           "algo": "rsa",
           "size": 2048
 },
 "names": [
       {
       "C": "CN",
       "L": "Beijing",
       "ST": "Beijing"
         }
     ]
}
EOF

#3.生成证书，ca-key.pem：根证书的私钥，ca.pem：ca根证书文件
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#4.指定 etcd三个节点之间的通信验证，server-csr.json是指定etcd三个节点之间的通信验证
cat > server-csr.json <<EOF
{
      "CN": "etcd",
      "hosts": [
     "192.168.58.10",
     "192.168.58.40",
     "192.168.58.50"
     ],
     "key": {
     "algo": "rsa",
     "size": 2048
     },
  "names": [
         {
           "C": "CN",
           "L": "BeiJing",
           "ST": "BeiJing"
       }
    ]
}
EOF
#5.生成 ETCD证书 server-key.pem server.pem
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

配置etcd服务脚本

cat etcd.sh
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.58.10 etcd02=https://192.168.58.40:2380,etcd03=https://192.168.58.50:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/k8s/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd

配置证书

mkdir -p /root/k8s/etcd-cert	#准备临时存放证书目录
mv etcd-cert.sh /root/k8s/etcd-cert
sh etcd-cert.sh
ls
ca-config.json  ca-csr.json  ca.pem        server.csr       server-key.pem
ca.csr          ca-key.pem   etcd-cert.sh  server-csr.json  server.pem

创建etcd工作目录，用于存放配置文件、命令、证书

mkdir -p /k8s/etcd/{cfg,bin,ssl}
cd /root/k8s/etcd-cert/
mv *.pem /k8s/etcd/ssl/		#将ca和etcd服务的证书和密钥放进去

配置etcd命令

wget -o /root/k8s/ https://github.com/etcd-io/etcd/releases/etcd-v3.3.10-linux-amd64.tar.gz
tar -zxvf etcd-v3.3.10-linux-amd64.tar.gz
cd etcd-v3.3.10-linux-amd64
mv etcd /k8s/etcd/bin/
mv etcdctl /k8s/etcd/bin

一键部署etcd集群

cd /root/k8s/
sh etcd.sh etcd01 192.168.58.10 etcd02=https://192.168.58.40:2380,etcd03=https://192.168.58.50:2380
##此时任务进入等待状态，等待其他etcd节点加入，查找不到5分钟后默认退出

另起一个etcd01终端, 查看生成的服务启动脚本和配置文件，查看服务启动状态，将etcd现成的工作目录和服务启动脚本拷贝给其他etcd节点

ls /usr/lib/systemd/system/ | grep etcd
etcd.service
ls /k8s/etcd/cfg
etcd
ps -ef | grep etcd
#端口，2379是提供给外部端口，2380是内部集群通讯端口，最多65536端口
scp -r /k8s/etcd/ root@192.168.58.40:/k8s/etcd/
scp -r /k8s/etcd/ root@192.168.58.50:/k8s/etcd/
scp /usr/lib/systemd/system/etcd.service root@192.168.58.40:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@192.168.58.50:/usr/lib/systemd/system/

修改其他etcd节点的etcd配置文件并启动服务

#etcd02节点上
vi /k8s/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.58.40:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.58.40:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.58.40:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.58.40:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.58.10:2380,etcd02=https://192.168.58.40:2380,etcd03=https://192.168.58.50:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

systemctl start etcd

#etcd03节点上
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.58.50:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.58.50:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.58.50:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.58.50:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.58.10:2380,etcd02=https://192.168.58.40:2380,etcd03=https://192.168.58.50:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

systemctl start etcd

此时主节点etcd01找到其他所有节点，等待结束，执行etcd集群健康检查

##指定证书文件时，若在证书目录下，可使用相对路径
/k8s/etcd/bin/etcdctl \
--ca-file=/k8s/etcd/ssl/ca.pem \
--cert-file=/k8s/etcd/ssl/server.pem --key-file=/k8s/etcd/ssl/server-key.pem \
--endpoints="https://192.168.58.10:2379,https://192.168.58.40:2379,https://192.168.58.50:2379" \
cluster-health

is healthy表示集群健康，etcd集群组建完成

3.docker引擎部署

所有node节点需要部署docker-ce
部署方法查看之前博客

4.flannel网络部署

所有node节点搭建flannel网络，不同pod间通讯需要此网络环境

在etcd中写入需要配置的flannel网络信息

##证书为相对路径，需在etcd的证书目录下执行
/k8s/etcd/bin/etcdctl \
--ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem \
--endpoints="https://192.168.58.10:2379,https://192.168.58.40:2379,https://192.168.58.50:2379" \
set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
##Network子网段为docker0的16位子网段

##查看etcd写入信息
/k8s/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://14.0.0.50:2379,https://14.0.0.60:2379,https://14.0.0.70:2379" get /coreos.com/network/config
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}

在两个node节点部署flannel软件
下载解压

wget https://github.com/flannel-io/flannel/releases/download/v0.10.0/flannel-v0.10.0-linux-amd64.tar.gz
tar -zxvf flannel-v0.10.0-linux-amd64.tar.gz
rm -f README.md		#介绍文档，删除多余文件

创建工作目录并拷贝解压出来的命令工具

mkdir -p /k8s/flannel/{cfg,bin,ssl}
mv mk-docker-opts.sh /k8s/flannel/bin/
mv flanneld /k8s/flannel/bin/

脚本配置flannel

vi flannel.sh
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
cat <<EOF >/k8s/flannel/cfg/flanneld        	#生成配置文件
FLANNEL_OPTIONS="-- etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/k8s/etcd/ssl/ca.pem \
-etcd-certfile=/k8s/etcd/ssl/server.pem \
-etcd-keyfile=/k8s/etcd/ssl/server-key.pem"
EOF
cat <<EOF >/usr/lib/systemd/system/flanneld.service          #生成服务启动脚本
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/k8s/flannel/cfg/flanneld
ExecStart=/k8s/flannel/bin/flanneld -- ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/k8s/flannel/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld

sh flannel.sh https://192.168.58.10:2379,https://192.168.58.40:2379,https://192.168.58.50:2379

通过ifconfig查看，此时新增了名为flannel的虚拟网卡

修改docker的服务启动文件指定子网段，将flannel与docker进行连接操作

vi /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// -- containerd=/run/containerd/containerd.sock
##加入$DOCKER_NETWORK_OPTIONS变量，该变量在flannel的网络环境变量文件中已定义

cat /run/flannel/subnet.env
DOCKER_OPT_BIP="--bip=172.17.62.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=true"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_NETWORK_OPTIONS=" --bip=172.17.62.1/24 --ip-masq=true --mtu=1450"

#重启docker服务配置生效
systemctl restart docker

5.部署master

创建kubernetes主节点相关组件服务证书
创建临时存放目录及工作目录

mkdir -p /root/k8s/k8s-cert
mkdir -p /k8s/kubernetes/{cfg,bin,ssl}

创建证书制作脚本

vi /root/k8s/k8s-cert/k8s-cert.sh
cat > ca-config.json <<EOF          #ca证书配置文件
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json <<EOF      #ca证书签名文件
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
      	    "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -        #生成ca.pem和ca-key.pem（CA认证机构）
cat > server-csr.json <<EOF
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "192.168.58.10",       #master1
      "192.168.58.60",       #master2，考虑到后面会增加master节点，所以添加一个master2的IP
      "192.168.58.200",      #集群对外提供的虚拟IP地址
      "192.168.58.70",       #代理端节点IP地址（master）
      "192.168.58.80",       #代理端节点IP地址（backup）
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server          #生成kube-apiserver的tls认证证书和认证私钥server.pem和server-key.pem
cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin          #kubectl的TLS认证证书和认证私钥，具有admin权限，admin.pem和admin-key.pem     
cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy		#生成kube-proxy的tls认证证书和认证私钥kube-proxy-key.pem和kube-proxy.pem

执行证书制作脚本并将证书加入工作目录

chmod +x k8s-cert.sh
sh k8s-cert.sh
cp ca*.pem server*.pem /k8s/kubernetes/ssl/		#拷贝ca密钥证书和api-server的密钥证书

配置master节点服务
解压k8s压缩包并将命令工具拷贝到工作目录

tar -zxvf kubernetes-server-linux-amd64.tar.gz
cd /root/k8s/kubernetes/server/bin/
cp kube-apiserver kubectl kube-controller-manager kube-scheduler /k8s/kubernetes/bin/

生成群集用户

#随机生成序列号
head -c 16 /dev/urandom | od -An -t x | tr -d ' '
#编写用户描述文件,填写内容：序列号，用户名，uid，用户组
vi /k8s/kubernetes/cfg/token.csv
b45d07d50eb7d9271fbc0f5530cf2d04,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
##该文件为一个用户的描述文件，基本格式为 Token,用户名,UID,用户组；这个文件在 apiserver 启动时被 apiserver 加载，然后就相当于在集群内创建了一个这个用户；接下来就可以用 RBAC 给他授权。

编辑apiserver配置脚本

vi /root/k8s/apiserver.sh
#!/bin/bash
# 其中需要指定master节点IP地址和etcd的群集
MASTER_ADDRESS=$1
ETCD_SERVERS=$2
#生成apiserver的配置文件
cat <<EOF >/k8s/kubernetes/cfg/kube-apiserver

KUBE_APISERVER_OPTS="--logtostderr=true \\
--v=4 \\
--etcd-servers=${ETCD_SERVERS} \\
--bind-address=${MASTER_ADDRESS} \\
--secure-port=6443 \\
--advertise-address=${MASTER_ADDRESS} \\
--allow-privileged=true \\
--service-cluster-ip-range=10.0.0.0/24 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--kubelet-https=true \\
--enable-bootstrap-token-auth \\
--token-auth-file=/k8s/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-50000 \\
--tls-cert-file=/k8s/kubernetes/ssl/server.pem  \\
--tls-private-key-file=/k8s/kubernetes/ssl/server-key.pem \\
--client-ca-file=/k8s/kubernetes/ssl/ca.pem \\
--service-account-key-file=/k8s/kubernetes/ssl/ca-key.pem \\
--etcd-cafile=/k8s/etcd/ssl/ca.pem \\
--etcd-certfile=/k8s/etcd/ssl/server.pem \\
--etcd-keyfile=/k8s/etcd/ssl/server-key.pem"

EOF

#生成api server的启动脚本
cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/k8s/kubernetes/cfg/kube-apiserver
ExecStart=/k8s/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver

执行脚本并检查服务启动状况

chmod +x apiserver.sh
sh apiserver.sh 192.168.58.10 https://192.168.58.10:2379,https://192.168.58.40:2379,https://192.168.58.50:2379

ps -ef | grep kube
netstat -napt | grep 6443	#监听的https端口
netstat -napt | grep 8080	#监听的http端口

编辑scheduler配置脚本

vi /root/k8s/scheduler.sh
#!/bin/bash

#指定监听本机的8080端口
MASTER_ADDRESS=$1

cat <<EOF >/k8s/kubernetes/cfg/kube-scheduler

KUBE_SCHEDULER_OPTS="--logtostderr=true \\
--v=4 \\
--master=${MASTER_ADDRESS}:8080 \\
--leader-elect"

EOF

#配置scheduler的启动脚本
cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/k8s/kubernetes/cfg/kube-scheduler
ExecStart=/k8s/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl restart kube-scheduler

执行脚本并查看集群健康状态

chmod +x scheduler.sh
sh scheduler.sh 127.0.0.1

/k8s/kubernetes/bin/kubectl get cs
显示scheduler为healthy

编辑controller-manager配置脚本

#!/bin/bash

MASTER_ADDRESS=$1

cat <<EOF >/k8s/kubernetes/cfg/kube-controller-manager


KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \\
--v=4 \\
--master=${MASTER_ADDRESS}:8080 \\
--leader-elect=true \\
--address=127.0.0.1 \\
--service-cluster-ip-range=10.0.0.0/24 \\
--cluster-name=kubernetes \\
--cluster-signing-cert-file=/k8s/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/k8s/kubernetes/ssl/ca-key.pem  \\
--root-ca-file=/k8s/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/k8s/kubernetes/ssl/ca-key.pem \\
--experimental-cluster-signing-duration=87600h0m0s"

EOF

cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/k8s/kubernetes/cfg/kube-controller-manager
ExecStart=/k8s/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager

执行脚本并进行集群健康检查

chmod +x controller-manager.sh
sh controller-manager.sh 127.0.0.1

/k8s/kubernetes/bin/kubectl get cs
显示controller-manager为healthy

master节点搭建完毕

6.部署node节点

将master节点k8s包内的kubelet和kube-proxy拷贝到node节点上

#node1和node2上
mkdir -p /k8s/kubernetes/{cfg,bin,ssl}
#master节点操作
cd /root/k8s/kubernetes/server/bin/
scp kubelet kube-proxy root@192.168.58.40:/k8s/kubernetes/bin/
scp kubelet kube-proxy root@192.168.58.50:/k8s/kubernetes/bin/

在master节点上编辑kubeconfig脚本

vi /root/k8s/kubeconfig/kubeconfig
APISERVER=$1
SSL_DIR=$2

# 创建kubelet bootstrapping kubeconfig 
export KUBE_APISERVER="https://$APISERVER:6443"

# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=$SSL_DIR/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=bootstrap.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=b45d07d50eb7d9271fbc0f5530cf2d04 \
  --kubeconfig=bootstrap.kubeconfig
#使用之前创建集群用户时使用的用户和token序列
# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig

# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

#----------------------

# 创建kube-proxy kubeconfig文件

kubectl config set-cluster kubernetes \
  --certificate-authority=$SSL_DIR/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \
  --client-certificate=$SSL_DIR/kube-proxy.pem \
  --client-key=$SSL_DIR/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

为便于命令的使用，将命令加入环境变量

##master上
vi /etc/profile		#在最后一行添加
export PATH=$PATH:/k8s/kubernetes/bin

source /etc/profile

执行kubeconfig脚本

##master上执行
sh kubeconfig 192.168.58.10 /root/k8s/k8s-cert

将生成的bootstrap.kubeconfig和kube-proxy.kubeconfig发送到node节点中去

cd /root/k8s/kubeconfig/
scp bootstrap.kubeconfig kube-proxy.kubeconfig root@192.168.58.40:/k8s/kubernetes/cfg/
scp bootstrap.kubeconfig kube-proxy.kubeconfig root@192.168.58.50:/k8s/kubernetes/cfg/

在master节点创建一个用户bootstrap用于apiserver的签名请求

kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

编辑kubelet的配置脚本

vim kubelet.sh
#!/bin/bash

NODE_ADDRESS=$1
DNS_SERVER_IP=${2:-"10.0.0.2"}

cat <<EOF >/k8s/kubernetes/cfg/kubelet

KUBELET_OPTS="--logtostderr=true \\
--v=4 \\
--hostname-override=${NODE_ADDRESS} \\
--kubeconfig=/k8s/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/k8s/kubernetes/cfg/bootstrap.kubeconfig \\
--config=/k8s/kubernetes/cfg/kubelet.config \\
--cert-dir=/k8s/kubernetes/ssl \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"

EOF

cat <<EOF >/k8s/kubernetes/cfg/kubelet.config

kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: ${NODE_ADDRESS}
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- ${DNS_SERVER_IP} 
clusterDomain: cluster.local.
failSwapOn: false
authentication:
  anonymous:
    enabled: true
EOF

cat <<EOF >/usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=/k8s/kubernetes/cfg/kubelet
ExecStart=/k8s/kubernetes/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet

执行脚本配置kubelet

#node1
chmod +x kubelet.sh
sh kubelet.sh 192.168.58.40
#node2
chmod +x kubelet.sh
sh kubelet.sh 192.168.58.50

此时node节点的kubelet会向apiserver请求颁发证书，需在master节点上同意请求

##master
kubectl get csr
node-csr-hqUd0OCtKa6V7JaY4ys9YphKnrjXHAPqEh47f4xF3fY   26s   kubelet-bootstrap   Pending
node-csr-9aS2UFdyjk-JarnJmH2sBXulm8QzYyPYXSm0ypS6JHo   12s     kubelet-bootstrap   Pending
##此时两条请求已发送过来，且为pending状态
#同意两个请求（命令+请求名称）
kubectl certificate approve node-csr-hqUd0OCtKa6V7JaY4ys9YphKnrjXHAPqEh47f4xF3fY
kubectl certificate approve node-csr-9aS2UFdyjk-JarnJmH2sBXulm8QzYyPYXSm0ypS6JHo
#此时再次查看两条请求
kubectl get csr
NAME                                                   AGE     REQUESTOR           CONDITION
node-csr-9aS2UFdyjk-JarnJmH2sBXulm8QzYyPYXSm0ypS6JHo   7m14s   kubelet-bootstrap   Approved,Issued
node-csr-hqUd0OCtKa6V7JaY4ys9YphKnrjXHAPqEh47f4xF3fY   9m58s   kubelet-bootstrap   Approved,Issued
##状态转为已同意
#此时即可看到节点信息
kubectl get nodes

编辑kube-proxy配置脚本

vi kube-proxy.sh
#!/bin/bash

NODE_ADDRESS=$1

cat <<EOF >/k8s/kubernetes/cfg/kube-proxy

KUBE_PROXY_OPTS="--logtostderr=true \\
--v=4 \\
--hostname-override=${NODE_ADDRESS} \\
--cluster-cidr=10.0.0.0/24 \\
--proxy-mode=ipvs \\
--kubeconfig=/k8s/kubernetes/cfg/kube-proxy.kubeconfig"

EOF

cat <<EOF >/usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=-/k8s/kubernetes/cfg/kube-proxy
ExecStart=/k8s/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy

执行kube-proxy配置脚本

#node1
chmod +x kube-proxy.sh
sh kube-proxy.sh 192.168.58.40
#node2
chmod +x kube-proxy.sh
sh kube-proxy.sh 192.168.58.50

以上node节点的kubelet和kube-proxy配置为两台机器都配置。
也可先只在node1配置，然后将配置文件和启动脚本拷贝给node2，然后在node2修改配置文件中的IP地址后重启服务

最终的节点状态检查，只要都是ready，即k8s集群配置成功