Kubernetes从kubeadm安装

环境

• k8s-master01 - master1：172.16.22.101（etcd-1）

• k8s-master02 - master2：172.16.22.102

• k8s-node01 - node1：172.16.22.103

• k8s-node02 - node2：172.15.22.104

• centos7.8

升级内核及公共配置

yum update -y
cat /etc/redhat-release 
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
yum --enablerepo=elrepo-kernel install kernel-ml -y

# 查看当前内核
uname -r 
# 查看已有的内核
egrep ^menuentry /etc/grub2.cfg | cut -f 2 -d \'

# 修改grub2启动新内核
grub2-set-default 0

systemctl stop firewalld && systemctl disable firewalld && iptables -F
setenforce 0 && sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

swapoff -a && sed -i 's#.*swap.*##g' /etc/fstab
free -m

cat <<\EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv6.conf.all.disable_ipv6 = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf

systemctl stop postfix && systemctl disable postfix

mkdir /var/log/journal
mkdir /etc/systemd/journald.conf.d
cat <<\EOF > /etc/systemd/journald.conf.d/99-prophet.conf
[Journal]
# 持久化保存到磁盘
Storage=persistent

# 压缩历史日志
Compress=yes

SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000

# 最大占用空间 5G
SystemMaxUse=5G

# 单日志文件最大 200M
SystemMaxFileSize=200M

# 日志保存时间 2 周
MaxRetentionSec=2week

# 不将日志发到syslog
ForwardToSyslog=no
EOF

systemctl restart systemd-journald

# kube-proxy开启ipvs的前置条件
modprobe br_netfilter

cat <<\EOF > /etc/sysconfig/modules/ipvs.modules 
#!/bin/bash
modprobe  -- ip_vs
modprobe  -- ip_vs_rr
modprobe  -- ip_vs_wrr
modprobe  -- ip_vs_sh
modprobe  -- nf_conntrack_ipv4
EOF

chmod 755 etc/sysconfig/modules/ipvs.modules 
bash /etc/sysconfig/modules/ipvs.modules 
lsmod | grep -e ip_vs -e nf_conntrack_ipv4


yum -y install yum-utils device-mapper-persistent-data lvm2
yum-config-manager \
--add-repo \
http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum -y update && yum -y install docker-ce
systemctl enable docker && systemctl start docker
docker info

cat << \EOF > /etc/docker/daemon.json
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m"
    }
}
EOF

mkdir -p /etc/systemd/system/docker.service.d

systemctl daemon-reload && systemctl restart docker && systemctl enable docker

cat <<\EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum -y update && yum -y install kubeadm kubectl kubelet 
systemctl enable kubelet

reboot

master01搭建

hostnamectl set-hostname k8s-master01

echo "172.16.22.101 k8s-master01" >> /etc/hosts
echo "172.16.22.102 k8s-master02" >> /etc/hosts
echo "172.16.22.103 k8s-node01" >> /etc/hosts
echo "172.16.22.104 k8s-node02" >> /etc/hosts

scp /etc/hosts [email protected]:/etc/hosts
scp /etc/hosts [email protected]:/etc/hosts
scp /etc/hosts [email protected]:/etc/hosts

# 导出集群初始化配置模板
kubeadm config print init-defaults > kubeadm-config.yaml

# 修改master节点地址
sed -i 's#advertiseAddress: .*#advertiseAddress: 172.16.22.101#g' kubeadm-config.yaml
# 修改Service网段，并添加Pod使用的地址段
sed -i 's#serviceSubnet:.*#podSubnet: 172.19.0.0/16\n  serviceSubnet: 172.18.0.0/24#g' kubeadm-config.yaml
# 修改成使用aliyun镜像
sed -i 's#imageRepository:.*#imageRepository: registry.aliyuncs.com/google_containers#g' kubeadm-config.yaml

# 添加应用ipvs配置
echo "---" >> kubeadm-config.yaml
echo "apiVersion: kubeproxy.config.k8s.io/v1alpha1" >> kubeadm-config.yaml
echo "kind: KubeProxyConfiguration" >> kubeadm-config.yaml
echo "featureGates:" >> kubeadm-config.yaml
echo "  SupportIPVSProxyMode: true" >> kubeadm-config.yaml
echo "mode: ipvs" >> kubeadm-config.yaml

# 初始化集群，并保存log至kubeadm-init.log
kubeadm init --config=kubeadm-config.yaml | tee kubeadm-init.log

# 安赵log提示修改配置
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

# 保留kubeadm-config.yaml  kubeadm-init.log文件
mkdir -p install-k8s/core
mv kubeadm-config.yaml  kubeadm-init.log install-k8s/core
mkdir -p install-k8s/plugin/flannel
cd install-k8s/plugin/flannel
yum -y install wget

# 安装flannel网络插件
# https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# 修改Pod使用的地址段
sed -i 's#"Network":.*#"Network": "172.19.0.0/16",#g' kube-flannel.yml
# 创建flannel的Pod
kubectl create -f kube-flannel.yml

加入Node节点

# 命令在kubeadm init日志/root/install-k8s/core/kubeadm-init.log的最后一行
kubeadm join 172.16.22.101:6443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:02d821acf1b0595943cb046a44356b68a85feb74e920b635713afc1fb732183f

鉴权（RBAC）

创建一个用户只能管理dev空间

export KUBE_APISERVER="https://172.16.22.101:6443"
export NEW_USERNAME="devuser"
export BIND_NS="dev"
export CFSSL_URL="https://pkg.cfssl.org/R1.2"
# 在Linux上创建新的用户
useradd ${NEW_USERNAME}
passwd ${NEW_USERNAME}
# 为新用户创建证书
mkdir -p ~/cert/${NEW_USERNAME}
# CN是用户名，O是用户组名，hosts不写代表任意主机
cat <<EOF > ~/cert/${NEW_USERNAME}/${NEW_USERNAME}-csr.json
{
  "CN": "${NEW_USERNAME}",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "GuangDong",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

# 安装CFSSL工具
yum install -y wget
wget "${CFSSL_URL}/cfssl_linux-amd64" -O /usr/local/bin/cfssl
wget "${CFSSL_URL}/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
wget "${CFSSL_URL}/cfssl-certinfo_linux-amd64" -O /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

# 创建用户证书
cd /etc/kubernetes/pki
cfssl gencert -ca=ca.crt -ca-key=ca.key -profile=kubernetes ~/cert/${NEW_USERNAME}/${NEW_USERNAME}-csr.json | cfssljson -bare ${NEW_USERNAME}
ll | grep ${NEW_USERNAME}

cd ~/cert/${NEW_USERNAME}/

# 配置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.crt \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${NEW_USERNAME}.kubeconfig

# 配置客户端认证参数
kubectl config set-credentials ${NEW_USERNAME} \
--client-certificate=/etc/kubernetes/pki/${NEW_USERNAME}.pem \
--client-key=/etc/kubernetes/pki/${NEW_USERNAME}-key.pem \
--embed-certs=true \
--kubeconfig=${NEW_USERNAME}.kubeconfig

# 如果没有相应的名称空间可以先创建
kubectl create namespace ${BIND_NS} 

# 设置上下文参数，将用户绑定到名称空间
kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=${NEW_USERNAME} \
--namespace=${BIND_NS} \
--kubeconfig=${NEW_USERNAME}.kubeconfig

# 让新用户可以在绑定的名称空间里做任何事
kubectl create rolebinding ${NEW_USERNAME}-admin-binding --clusterrole=admin --user=${NEW_USERNAME} --namespace=${BIND_NS}

# 让新用户可以直接使用kubectl
su - ${NEW_USERNAME}
mkdir .kube/
exit
cp ~/cert/${NEW_USERNAME}/${NEW_USERNAME}.kubeconfig /home/${NEW_USERNAME}/.kube/config
chown ${NEW_USERNAME}:${NEW_USERNAME} /home/${NEW_USERNAME}/.kube/config

# 设置默认配置文件
su - ${NEW_USERNAME}
kubectl config user-context kubernetes --kubeconfig=.kube/config
exit

修改证书年限

cd /etc/kubernetes/pki

# 查看ca证书使用年限
export PKI_CRT=ca
openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not Before" && openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not After"
# Not Before: Nov 26 07:18:20 2020 GMT
# Not After : Nov 24 07:18:20 2030 GMT


# API Server证书
export PKI_CRT=apiserver
openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not Before" && openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not After"
# Not Before: Nov 26 07:18:20 2020 GMT
# Not After : Nov 26 07:18:20 2021 GMT
# 如果一年更新一次集群，则可忽略该问题

# -----------------------------------------------------------------------------------------------------

# 安装Go环境
# https://golang.org/dl/go1.15.6.linux-amd64.tar.gz
wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz
tar zxvf go1.15.6.linux-amd64.tar.gz -C /usr/local/

echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile
source /etc/profile
go version

# 获取kubernetes源码
yum -y install git
git clone https://github.com/kubernetes/kubernetes.git
cd kubernetes
# 切换到和当前运行的kubernetes相同的版本
kubectl version
git checkout -b remotes/origin/release-1.19.4 v1.19.4

# 修改源码中的证书年限
# 1.14之前
# vim staging/src/k8s.io/client-go/util/cert/cert.go
# 1.14之后
vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go
# ?certTmpl
# NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity).UTC(),
# 修改 kubeadmconstants.CertificateValidity 为 time.Hour * 24 * 365 * 10
# NotAfter:     time.Now().Add(time.Hour * 24 * 365 * 10).UTC(),
# 修改为10年
# 保存退出

# 编译kubeadm源码
make WHAT=cmd/kubeadm GOFLAGS=-v

# 备份旧的kubeadm
mv /usr/bin/kubeadm /usr/bin/kubeadm.bak.old

# 用编译好的kubeadm代替旧的kubeadm
cp _output/bin/kubeadm /usr/bin/
chmod +x /usr/bin/kubeadm

# 备份集群现有证书
cp -r /etc/kubernetes/pki /etc/kubernetes/pki.bak.old

# 使用新的kubeadm更新集群所有证书（kubeadm-config.yaml文件是master01搭建时创建的配置文件）
cd ~
kubeadm alpha certs renew all --config=install-k8s/core/kubeadm-config.yaml

# -----------------------------------------------------------------------------------------------------

# 查看证书是否更新
cd /etc/kubernetes/pki
# 查看ca证书使用年限
export PKI_CRT=ca
openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not Before" && openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not After"
# Not Before: Nov 26 07:18:20 2020 GMT
# Not After : Nov 24 07:18:20 2030 GMT

# API Server证书
export PKI_CRT=apiserver
openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not Before" && openssl x509 -in ${PKI_CRT}.crt -text -noout | grep "Not After"
# Not Before: Nov 26 07:18:20 2020 GMT
# Not After : Dec  2 03:06:01 2030 GMT

高可用方案

1.14版本之后

添加master2：172.16.22.102
使用172.16.22.100作为VIP

搭建HAProxy负载均衡LB

master1和2都要做

搭建Keepalived（VRRP）

master1和2都要做

更新master1配置

cat ~/install-k8s/core/kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 172.16.22.101
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: k8s-master01
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
# 添加集群VIP
controlPlaneEndpoint: 172.16.22.100
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.19.0
networking:
  dnsDomain: cluster.local
  podSubnet: 172.19.0.0/16
  serviceSubnet: 172.18.0.0/24
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

添加master2到集群

先做 # 升级内核及公共配置