K8s——完整单节点二进制部署

K8s——完整单节点二进制部署

相关软件包及文档：
链接：https://pan.baidu.com/s/1l4vVCkZ03la-VpIFXSz1dA
提取码：rg99

实验步骤

-------etcd群集部署-------
1：自签ETCD证书
2：ETCD部署
3：Node安装docker
4：Flannel部署（先写入子网到etcd）
---------master----------
5：自签APIServer证书
6：部署APIServer组件（token，csv）
7：部署controller-manager（指定apiserver证书）和scheduler组件
----------node----------
8：生成kubeconfig（bootstrap，kubeconfig和kube-proxy.kubeconfig）
9：部署kubelet组件
10：部署kube-proxy组件
----------加入群集----------
11：kubectl get csr && kubectl certificate approve 允许办法证书，加入群集
12：添加一个node节点
13：查看kubectl get node 节点

环境准备

master节点：

CentOS 7-1：192.168.217.130

node节点：

CentOS 7-3：192.168.217.132 docker

CentOS 7-4：192.168.217.133 docker

etcd群集部署

master节点操作

关闭防火墙和安全功能
[root@localhost k8s]# systemctl stop firewalld.service 
[root@localhost k8s]# setenforce 0
[root@localhost ~]# mkdir k8s
[root@localhost ~]# cd k8s
[root@localhost k8s]# mkdir etcd-cert
[root@localhost k8s]# mv etcd-cert.sh etcd-cert
[root@localhost k8s]# ls
etcd-cert  etcd.sh
[root@localhost k8s]# vim cfssl.sh
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
//下载cfssl官方包
[root@localhost k8s]# bash cfssl.sh
[root@localhost k8s]# ls /usr/local/bin/
cfssl  cfssl-certinfo  cfssljson
//cfssl 生成证书工具   cfssljson通过传入json文件生成证书
  cfssl-certinfo查看证书信息
[root@localhost k8s]# cd etcd-cert/
//定义CA证书
[root@localhost etcd-cert]# cat > ca-config.json <
 {
   "signing":{
     "default":{
       "expiry":"87600h"
     },
     "profiles":{
       "www":{
         "expiry":"87600h",
         "usages":[
           "signing",
           "key encipherment",
           "server auth",
           "client auth"
         ]
       }
     }
   }
 }
 EOF
 
[root@localhost etcd-cert]# ls
ca-config.json  etcd-cert.sh
//实现证书签名
[root@localhost etcd-cert]# cat > ca-csr.json <
 {   
     "CN": "etcd CA",
     "key": {
         "algo": "rsa",
         "size": 2048
     },
     "names": [
         {
             "C": "CN",
             "L": "Beijing",
             "ST": "Beijing"
         }
     ]
 }
 EOF
[root@localhost etcd-cert]# ls
ca-config.json  ca-csr.json  etcd-cert.sh
//生成证书，生成ca-key.pem  ca.pem
[root@localhost etcd-cert]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2020/02/09 12:02:25 [INFO] generating a new CA key and certificate from CSR
2020/02/09 12:02:25 [INFO] generate received request
2020/02/09 12:02:25 [INFO] received CSR
2020/02/09 12:02:25 [INFO] generating key: rsa-2048
2020/02/09 12:02:25 [INFO] encoded CSR
2020/02/09 12:02:25 [INFO] signed certificate with serial number 414178568497244453573198793884960056974675134689
//指定etcd三个节点之间的通信验证
[root@localhost etcd-cert]# cat > server-csr.json <
 {
     "CN": "etcd",
     "hosts": [
     "192.168.217.130",
     "192.168.217.132", 
     "192.168.217.133"
     ],
     "key": {
         "algo": "rsa",
         "size": 2048
     },
     "names": [
         {
             "C": "CN",
            "L": "BeiJing",
             "ST": "BeiJing"
         }
     ]
 }
 EOF
 
 //生成ETCD证书 server-key.pem   server.pem
[root@localhost etcd-cert]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2020/02/09 12:04:11 [INFO] generate received request
2020/02/09 12:04:11 [INFO] received CSR
2020/02/09 12:04:11 [INFO] generating key: rsa-2048
2020/02/09 12:04:11 [INFO] encoded CSR
2020/02/09 12:04:11 [INFO] signed certificate with serial number 39681540923464063462555913584059096025181981744
2020/02/09 12:04:11 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

上传下列三个压缩包

继续在master节点上操作

[root@localhost etcd-cert]# ls
ca-config.json  etcd-cert.sh                          server-csr.json
ca.csr          etcd-v3.3.10-linux-amd64.tar.gz       server-key.pem
ca-csr.json     flannel-v0.10.0-linux-amd64.tar.gz    server.pem
ca-key.pem      kubernetes-server-linux-amd64.tar.gz
ca.pem          server.csr
[root@localhost etcd-cert]# mv *.tar.gz ../
[root@localhost etcd-cert]# cd ../
[root@localhost k8s]# ls
etcd-cert                        flannel-v0.10.0-linux-amd64.tar.gz
etcd.sh                          kubernetes-server-linux-amd64.tar.gz
etcd-v3.3.10-linux-amd64.tar.gz
[root@localhost k8s]# tar zxvf etcd-v3.3.10-linux-amd64.tar.gz 

[root@localhost k8s]# ls etcd-v3.3.10-linux-amd64
Documentation  etcdctl            README.md
etcd           README-etcdctl.md  READMEv2-etcdctl.md
[root@localhost k8s]# mkdir /opt/etcd/{cfg,bin,ssl} -p
[root@localhost k8s]# mv etcd-v3.3.10-linux-amd64/etcd etcd-v3.3.10-linux-amd64/etcdctl /opt/etcd/bin/
//证书拷贝
[root@localhost k8s]#  cp etcd-cert/*.pem /opt/etcd/ssl/
//进入卡住状态等待其他节点加入
[root@localhost k8s]# bash etcd.sh etcd01 192.168.217.130 etcd02=https://192.168.217.132:2380,etcd03=https://192.168.217.133:2380

//使用另外一个会话打开，会发现etcd进程已经开启
[root@master ~]# ps -ef | grep etcd
root       3479   1780  0 11:48 pts/0    00:00:00 bash etcd.sh etcd01 192.168.18.128 etcd02=https://192.168.195.148:2380,etcd03=https://192.168.217.130:2380
root       3530   3479  0 11:48 pts/0    00:00:00 systemctl restart etcd
root       3540      1  1 11:48 ?        00:00:00 /opt/etcd/bin/etcd 

//拷贝证书去其他节点
[root@localhost k8s]# scp -r /opt/etcd/ [email protected]:/opt/
[root@localhost k8s]# scp -r /opt/etcd/ [email protected]:/opt/
[root@localhost ~]# scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
[root@localhost ~]# scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/

在node1节点修改

[root@localhost ~]# vim /opt/etcd/cfg/etcd

#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.217.132:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.217.132:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.217.132:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.217.132:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.217.130:2380,etcd02=https://192.168.217.132:2380,etcd03=https://192.168.217.133:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
//关闭防火墙和安全功能
[root@localhost ~]# systemctl stop firewalld.service 
[root@localhost ~]# setenforce 0
//启动
[root@localhost ~]# systemctl start etcd

在node2节点修改

[root@localhost ~]# vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.217.133:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.217.133:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.217.133:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.217.133:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.217.130:2380,etcd02=https://192.168.217.132:2380,etcd03=https://192.168.217.133:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
//关闭防火墙和安全功能
[root@localhost ~]# systemctl stop firewalld.service 
[root@localhost ~]# setenforce 0
//启动
[root@localhost ~]# systemctl start etcd

在master主键上查看集群状态

[root@localhost k8s]# cd etcd-cert/
[root@localhost etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379" cluster-health
member 1e7e5a367520c0a0 is healthy: got healthy result from https://192.168.217.132:2379
member 8acf5a147f31a409 is healthy: got healthy result from https://192.168.217.130:2379
member 92d3b70800f20058 is healthy: got healthy result from https://192.168.217.133:2379
cluster is healthy

docker引擎部署

所有node节点部署docker引擎

#安装依懒包
yum install -y yum-utils device-mapper-persistent-data lvm2
#设置阿里云镜像版
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#安装docker-ce
yum install -y docker-ce
#关闭防火墙和安全功能
systemctl stop firewalld.service
setenforce 0
#开启docker，并设置为开机自启
systemctl start docker.service
systemctl enable docker.service
#镜像加速，加速服务在阿里云平台上获取
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://dticwlxc.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker
-----网络优化---
vim /etc/sysctl.conf
net.ipv4.ip_forward=1
sysctl -p
service network restart
systemctl restart docker

flanner网络配置、

在master主键上
//写入分配的子网段到ETCD中，供flannel使用
[root@localhost etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168217.133:2379" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}

//查看写入的信息
[root@localhost etcd-cert]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379" get /coreos.com/network/config
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
//拷贝到所有node节点（只需要部署在node节点即可）
[root@localhost k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz [email protected]:/root
[root@localhost k8s]# scp flannel-v0.10.0-linux-amd64.tar.gz [email protected]:/root

node节点的操作
把flannel.sh包拷贝到node节点上

//所有node节点操作解压
[root@localhost ~]# tar zxvf flannel-v0.10.0-linux-amd64.tar.gz 
flanneld
mk-docker-opts.sh
README.md
//创建k8s工作目录
[root@localhost ~]# mkdir /opt/kubernetes/{cfg,bin,ssl} -p
[root@localhost ~]# mv mk-docker-opts.sh flanneld /opt/kubernetes/bin/
[root@localhost ~]# vim flannel.sh
#!/bin/bash

ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}

cat </opt/kubernetes/cfg/flanneld

FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/opt/etcd/ssl/ca.pem \
-etcd-certfile=/opt/etcd/ssl/server.pem \
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"

EOF

cat </usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld

//开启flannel网络功能
[root@localhost ~]# bash flannel.sh https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379

Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service.

//配置docker连接flannel
[root@localhost ~]# vim /usr/lib/systemd/system/docker.service
#service段落做如下改动
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/subnet.env   
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

[root@localhost ~]# cat /run/flannel/subnet.env
DOCKER_OPT_BIP="--bip=172.17.24.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
//说明：bip指定启动时的子网
DOCKER_NETWORK_OPTIONS=" --bip=172.17.24.1/24 --ip-masq=false --mtu=1450" 

//重启docker服务
[root@localhost ~]# systemctl daemon-reload
[root@localhost ~]# systemctl restart docker


//查看flannel网络
[root@localhost ~]# ifconfig
flannel.1: flags=4163,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 172.17.42.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::fc7c:e1ff:fe1d:224  prefixlen 64  scopeid 0x20
        ether fe:7c:e1:1d:02:24  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 26 overruns 0  carrier 0  collisions 0

//测试ping通对方docker0网卡 证明flannel起到路由作用

//安装docker容器
[root@localhost ~]# docker run -it centos:7 /bin/bash

[root@5f9a65565b53 /]# yum install net-tools -y

//再次测试ping通两个node中的centos:7容器

部署master主键

//在master上操作，api-server生成证书
[root@localhost k8s]# unzip master.zip
Archive:  master.zip
  inflating: apiserver.sh
  inflating: controller-manager.sh
  inflating: scheduler.sh
[root@localhost k8s]# mkdir /opt/kubernetes/{cfg,bin,ssl} -p
[root@localhost k8s]# mkdir k8s-cert
[root@localhost k8s]# cd k8s-cert/
[root@localhost k8s-cert]# ls  #需要上传k8s-cert.sh到此目录下
k8s-cert.sh
[root@localhost k8s-cert]# vim k8s-cert.sh 
cat > ca-config.json <{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json <{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
      	    "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

#-----------------------
#此处下列地址要对应我们的所有地址
cat > server-csr.json <{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "192.168.217.130",  //master1
      "192.168.217.131",  //master2
      "192.168.217.100",  //vip
      "192.168.217.136",  //lb （master）
      "192.168.217.139",  //lb （backup）
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

#-----------------------

cat > admin-csr.json <{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

#-----------------------

cat > kube-proxy-csr.json <{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

//生成k8s证书
[root@localhost k8s-cert]# bash k8s-cert.sh
2020/02/09 13:21:50 [INFO] generating a new CA key and certificate from CSR
2020/02/09 13:21:50 [INFO] generate received request
2020/02/09 13:21:50 [INFO] received CSR
2020/02/09 13:21:50 [INFO] generating key: rsa-2048
2020/02/09 13:21:52 [INFO] encoded CSR
2020/02/09 13:21:52 [INFO] signed certificate with serial number 282811744124251466915241505007041774312094040461
2020/02/09 13:21:52 [INFO] generate received request
2020/02/09 13:21:52 [INFO] received CSR
2020/02/09 13:21:52 [INFO] generating key: rsa-2048
2020/02/09 13:21:52 [INFO] encoded CSR
2020/02/09 13:21:52 [INFO] signed certificate with serial number 557251511759469661168687407003875720478633099309
2020/02/09 13:21:52 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
2020/02/09 13:21:52 [INFO] generate received request
2020/02/09 13:21:52 [INFO] received CSR
2020/02/09 13:21:52 [INFO] generating key: rsa-2048
2020/02/09 13:21:52 [INFO] encoded CSR
2020/02/09 13:21:52 [INFO] signed certificate with serial number 434232425904946706700312747715481558517023670923
2020/02/09 13:21:52 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
2020/02/09 13:21:52 [INFO] generate received request
2020/02/09 13:21:52 [INFO] received CSR
2020/02/09 13:21:52 [INFO] generating key: rsa-2048
2020/02/09 13:21:53 [INFO] encoded CSR
2020/02/09 13:21:53 [INFO] signed certificate with serial number 421931052064866614655634756727158680028382359801
2020/02/09 13:21:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").


[root@localhost k8s-cert]# ls *pem
admin-key.pem  ca-key.pem  kube-proxy-key.pem  server-key.pem
admin.pem      ca.pem      kube-proxy.pem      server.pem
[root@localhost k8s-cert]# cp ca*pem server*pem /opt/kubernetes/ssl/
[root@localhost k8s-cert]# cd ..
//解压kubernetes压缩包
[root@localhost k8s]# tar zxvf kubernetes-server-linux-amd64.tar.gz 

[root@localhost k8s]# cd /root/k8s/kubernetes/server/bin
//复制关键命令文件
[root@localhost bin]# cp kube-apiserver kubectl kube-controller-manager kube-scheduler /opt/kubernetes/bin/
[root@localhost bin]# cd /root/k8s
//随机生成序列号
[root@localhost k8s]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
00e6bc07f3ba1318785bfa2942e55ec2
[root@localhost k8s]#  vim /opt/kubernetes/cfg/token.csv
00e6bc07f3ba1318785bfa2942e55ec2,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
序列号，用户名，id，角色

//二进制文件，token，证书都准备好，开启apiserver
[root@localhost k8s]# bash apiserver.sh 192.168.217.130 https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.

//查看配置文件
[root@localhost k8s]# cat /opt/kubernetes/cfg/kube-apiserver 
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379 \
--bind-address=192.168.217.130 \
--secure-port=6443 \
--advertise-address=192.168.217.130 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem"

//监听的https端口
[root@localhost k8s]# netstat -ntap | grep 6443
tcp        0      0 192.168.217.130:6443    0.0.0.0:*               LISTEN      43188/kube-apiserve 
tcp        0      0 192.168.217.130:6443    192.168.217.130:32864   ESTABLISHED 43188/kube-apiserve 
tcp        0      0 192.168.217.130:32864   192.168.217.130:6443    ESTABLISHED 43188/kube-apiserve 
[root@localhost k8s]# netstat -ntap | grep 8080
tcp        0      0 127.0.0.1:8080          0.0.0.0:*               LISTEN      43188/kube-apiserve 

//启动scheduler服务
[root@localhost k8s]#  ./scheduler.sh 127.0.0.1
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
[root@localhost k8s]#  ps aux | grep ku
postfix   43016  0.0  0.1  91732  4008 ?        S    13:24   0:00 pickup -l -t unix -u
root      43188 28.1  8.1 398892 315152 ?       Ssl  13:31   0:20 /opt/kubernetes/bin/kube-apiserver --logtostderr=true --v=4 --etcd-servers=https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379 --bind-address=192.168.217.130 --secure-port=6443 --advertise-address=192.168.217.130 --allow-privileged=true --service-cluster-ip-range=10.0.0.0/24 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction --authorization-mode=RBAC,Node --kubelet-https=true --enable-bootstrap-token-auth --token-auth-file=/opt/kubernetes/cfg/token.csv --service-node-port-range=30000-50000 --tls-cert-file=/opt/kubernetes/ssl/server.pem --tls-private-key-file=/opt/kubernetes/ssl/server-key.pem --client-ca-file=/opt/kubernetes/ssl/ca.pem --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem --etcd-cafile=/opt/etcd/ssl/ca.pem --etcd-certfile=/opt/etcd/ssl/server.pem --etcd-keyfile=/opt/etcd/ssl/server-key.pem
root      43262  2.3  0.5  45616 21228 ?        Ssl  13:32   0:00 /opt/kubernetes/bin/kube-scheduler --logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect
root      43273  0.0  0.0 112680   984 pts/1    S+   13:32   0:00 grep --color=auto ku

//启动controller-manager
[root@localhost k8s]# chmod +x controller-manager.sh 
[root@localhost k8s]# ./controller-manager.sh 127.0.0.1
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.

//查看master 节点状态
[root@localhost k8s]# /opt/kubernetes/bin/kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}   
[root@localhost k8s]#

node节点部署

//master上操作
//把 kubelet、kube-proxy拷贝到node节点上去

[root@localhost bin]# scp kubelet kube-proxy [email protected]:/opt/kubernetes/bin/
root@192.168.217.132's password: 
kubelet                                            100%  168MB  17.4MB/s   00:09    
kube-proxy                                         100%   48MB  15.7MB/s   00:03    
[root@localhost bin]# scp kubelet kube-proxy [email protected]:/opt/kubernetes/bin/
[email protected]'s password: 
kubelet                                            100%  168MB  33.7MB/s   00:05    
kube-proxy                                         100%   48MB  48.0MB/s   00:01  
  
//在node1上操作（复制node.zip到/root目录下再解压）
[root@localhost ~]#  unzip node.zip
Archive:  node.zip
  inflating: proxy.sh                
  inflating: kubelet.sh

//在master上操作
[root@localhost bin]# cd /root/k8s/
[root@localhost k8s]# mkdir kubeconfig
[root@localhost k8s]# cd kubeconfig/
//拷贝kubeconfig.sh文件进行重命名
[root@localhost kubeconfig]# mv kubeconfig.sh kubeconfi
[root@localhost kubeconfig]# vim kubeconfi
删除以下部分
# 创建 TLS Bootstrapping Token
#BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
BOOTSTRAP_TOKEN=0fb61c46f8991b718eb38d27b605b008

cat > token.csv <{BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
//获取token信息
[root@localhost kubeconfig]# cat /opt/kubernetes/cfg/token.csv
00e6bc07f3ba1318785bfa2942e55ec2,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

//配置文件修改为tokenID
[root@localhost kubeconfig]# vim kubeconfi
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=00e6bc07f3ba1318785bfa2942e55ec2 \
  --kubeconfig=bootstrap.kubeconfig

//设置环境变量（可以写入到/etc/profile中
[root@localhost kubeconfig]# export PATH=$PATH:/opt/kubernetes/bin/
[root@localhost kubeconfig]#  kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}  
 
//生成配置文件
[root@localhost kubeconfig]# bash kubeconfi 192.168.217.130 /root/k8s/k8s-cert/
Cluster "kubernetes" set.
User "kubelet-bootstrap" set.
Context "default" created.
Switched to context "default".
Cluster "kubernetes" set.
User "kube-proxy" set.
Context "default" created.
Switched to context "default".
[root@localhost kubeconfig]# ls
bootstrap.kubeconfig  kubeconfi  kubeconfig  kube-proxy.kubeconfig

//拷贝配置文件到node节点
[root@localhost kubeconfig]# scp bootstrap.kubeconfig kube-proxy.kubeconfig [email protected]:/opt/kubernetes/cfg/

[root@localhost kubeconfig]# scp bootstrap.kubeconfig kube-proxy.kubeconfig [email protected]:/opt/kubernetes/cfg/
  
 //创建bootstrap角色赋予权限用于连接apiserver请求签名（关键
[root@localhost kubeconfig]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created

node1节点部署

 //    在node1节点上操作         
[root@localhost ~]# bash kubelet.sh 192.168.217.132
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
//检查kubelet服务启动
[root@localhost ~]# ps aux | grep kube
root      45482  0.0  0.8 300512 16120 ?        Ssl  12:47   0:01 /opt/kubernetes/bin/flanneld --ip-masq --etcd-endpoints=https://192.168.217.130:2379,https://192.168.217.132:2379,https://192.168.217.133:2379 -etcd-cafile=/opt/etcd/ssl/ca.pem -etcd-certfile=/opt/etcd/ssl/server.pem -etcd-keyfile=/opt/etcd/ssl/server-key.pem
root      51094  3.8  2.3 370688 43796 ?        Ssl  13:51   0:00 /opt/kubernetes/bin/kubelet --logtostderr=true --v=4 --hostname-override=192.168.217.132 --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig --bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig --config=/opt/kubernetes/cfg/kubelet.config --cert-dir=/opt/kubernetes/ssl --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0
root      51126  0.0  0.0 112676   980 pts/1    R+   13:51   0:00 grep --color=auto kube


//在master上操作
//检查到node01节点的请求
[root@localhost kubeconfig]# kubectl get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-gqmSZ0G1gKFEY_-riyJr0oPnZ7LrrN45VGgHLsmFNuE   42s   kubelet-bootstrap   Pending    #等待集群给该节点颁发证书

[root@localhost kubeconfig]# kubectl certificate approve node-csr-gqmSZ0G1gKFEY_-riyJr0oPnZ7LrrN45VGgHLsmFNuE
certificatesigningrequest.certificates.k8s.io/node-csr-gqmSZ0G1gKFEY_-riyJr0oPnZ7LrrN45VGgHLsmFNuE approved
//继续查看证书状态
[root@localhost kubeconfig]# kubectl get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-gqmSZ0G1gKFEY_-riyJr0oPnZ7LrrN45VGgHLsmFNuE   79s   kubelet-bootstrap   Approved,Issued   （已经被允许加入群集


//查看群集节点，成功加入node01节点
[root@localhost kubeconfig]# kubectl get node
NAME              STATUS   ROLES    AGE   VERSION
192.168.217.132   Ready       51s   v1.12.3


//在node01节点操作，启动proxy服务
[root@localhost ~]# bash proxy.sh 192.168.217.132
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.
[root@localhost ~]# systemctl status kube-proxy.service
● kube-proxy.service - Kubernetes Proxy
   Loaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: disabled)
   Active: active (running) since 日 2020-02-09 13:54:00 CST; 14s ago
 Main PID: 51694 (kube-proxy)
   Memory: 9.3M
   CGroup: /system.slice/kube-proxy.service
           ‣ 51694 /opt/kubernetes/bin/kube-proxy --logtostderr=true --v=4 --hostn...

2月 09 13:54:04 localhost.localdomain kube-proxy[51694]: I0209 13:54:04.951041   ...
2月 09 13:54:05 localhost.localdomain kube-proxy[51694]: I0209 13:54:05.150004   ...
2月 09 13:54:06 localhost.localdomain kube-proxy[51694]: I0209 13:54:06.961409   ...
2月 09 13:54:07 localhost.localdomain kube-proxy[51694]: I0209 13:54:07.162451   ...
2月 09 13:54:08 localhost.localdomain kube-proxy[51694]: I0209 13:54:08.973883   ...
2月 09 13:54:09 localhost.localdomain kube-proxy[51694]: I0209 13:54:09.173205   ...
2月 09 13:54:11 localhost.localdomain kube-proxy[51694]: I0209 13:54:11.012830   ...
2月 09 13:54:11 localhost.localdomain kube-proxy[51694]: I0209 13:54:11.184843   ...
2月 09 13:54:13 localhost.localdomain kube-proxy[51694]: I0209 13:54:13.026319   ...
2月 09 13:54:13 localhost.localdomain kube-proxy[51694]: I0209 13:54:13.196327   ...
Hint: Some lines were ellipsized, use -l to show in full.

node 2节点部署

//在node01节点操作
//把现成的/opt/kubernetes目录复制到其他节点进行修改即可
[root@localhost ~]# scp -r /opt/kubernetes/ [email protected]:/opt/
The authenticity of host '192.168.217.133 (192.168.217.133)' can't be established.
ECDSA key fingerprint is SHA256:AL1Fr3wJ/6yfYqW2lPhp7nUAVx+RiyeOKTTgyCralGE.
ECDSA key fingerprint is MD5:c9:45:35:f9:b6:bd:80:2b:8e:88:b4:01:38:ac:3c:87.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.217.133' (ECDSA) to the list of known hosts.
[email protected]'s password: 
flanneld                                           100%  241   387.7KB/s   00:00    
bootstrap.kubeconfig                               100% 2169     2.1MB/s   00:00    
kube-proxy.kubeconfig                              100% 6271     1.3MB/s   00:00    
kubelet                                            100%  379   154.8KB/s   00:00    
kubelet.config                                     100%  269   106.7KB/s   00:00    
kubelet.kubeconfig                                 100% 2298     1.3MB/s   00:00    
kube-proxy                                         100%  191   147.2KB/s   00:00    
mk-docker-opts.sh                                  100% 2139     1.6MB/s   00:00    
scp: /opt//kubernetes/bin/flanneld: Text file busy
kubelet                                            100%  168MB  44.3MB/s   00:03    
kube-proxy                                         100%   48MB  57.5MB/s   00:00    
kubelet.crt                                        100% 2197   134.4KB/s   00:00    
kubelet.key                                        100% 1675   173.8KB/s   00:00    
kubelet-client-2020-02-09-13-52-42.pem             100% 1277   882.3KB/s   00:00    
kubelet-client-current.pem                         100% 1277     1.1MB/s   00:00  

 //把kubelet，kube-proxy的service文件拷贝到node2中
[root@localhost ~]# scp /usr/lib/systemd/system/{kubelet,kube-proxy}.service [email protected]:/usr/lib/systemd/system/
root@192.168.217.133's password: 
kubelet.service                                    100%  264   113.5KB/s   00:00    
kube-proxy.service                                 100%  231   149.8KB/s   00:00    


//在node2 节点上操作，进行修改
[root@localhost ~]# unzip node.zip 
Archive:  node.zip
  inflating: proxy.sh                
  inflating: kubelet.sh
  
//首先删除复制过来的证书，等会node02会自行申请证书
[root@localhost ~]# cd /opt/kubernetes/ssl/
[root@localhost ssl]# rm -rf *
[root@localhost ssl]# cd ../cfg/

//修改配置文件kubelet  kubelet.config kube-proxy（三个配置文件）
[root@localhost cfg]#  vim kubelet
KUBELET_OPTS="--logtostderr=true \
--v=4 \
--hostname-override=192.168.217.133 \
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
--config=/opt/kubernetes/cfg/kubelet.config \
--cert-dir=/opt/kubernetes/ssl \
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"

[root@localhost cfg]# vim kubelet.config
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.217.133
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local.
failSwapOn: false
authentication:
  anonymous:
    enabled: true

[root@localhost cfg]# vim kube-proxy
KUBE_PROXY_OPTS="--logtostderr=true \
--v=4 \
--hostname-override=192.168.217.133 \
--cluster-cidr=10.0.0.0/24 \
--proxy-mode=ipvs \
--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig"

//启动服务
[root@localhost cfg]#  systemctl start kubelet.service
[root@localhost cfg]# systemctl enable kubelet.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
[root@localhost cfg]# systemctl start kube-proxy.service
[root@localhost cfg]# systemctl enable kube-proxy.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.

//在master节点上操作
[root@localhost kubeconfig]#  kubectl get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-DvP9rsDUlQAVuYDoirb-0KomiLAPK88AjyH07um-H70   42s   kubelet-bootstrap   Pending
node-csr-gqmSZ0G1gKFEY_-riyJr0oPnZ7LrrN45VGgHLsmFNuE   10m   kubelet-bootstrap   Approved,Issued

//授权许可加入群集
[root@localhost kubeconfig]# kubectl certificate approve node-csr-DvP9rsDUlQAVuYDoirb-0KomiLAPK88AjyH07um-H70
certificatesigningrequest.certificates.k8s.io/node-csr-DvP9rsDUlQAVuYDoirb-0KomiLAPK88AjyH07um-H70 approved

//查看群集中的节点
[root@localhost kubeconfig]# kubectl get node
NAME              STATUS     ROLES    AGE     VERSION
192.168.217.132   Ready         9m39s   v1.12.3
192.168.217.133   NotReady      6s      v1.12.3