【kubernetes】k8s单master集群环境搭建及kuboard部署

k8s入门学习环境搭建

学习于许大仙: https://www.yuque.com/fairy-era
k8s官网 https://kubernetes.io/
kuboard官网 https://kuboard.cn/
基于k8s 1.21.10版本

前置环境准备

一主两从，三台虚拟机

CPU	内存	硬盘	角色	主机名	IP	hostname	操作系统
4C	16G	50G	master	k8s-master	192.168.8.11	k8s-master	centos7.9
4C	16G	50G	worker(node)	k8s-node1	192.168.8.22	k8s-node1	centos7.9
4C	16G	50G	worker(node)	k8s-node2	192.168.8.33	k8s-node2	centos7.9

系统内核版本升级

#查看内核版本，太低建议升级，具体操作详见许大仙笔记
cat /etc/redhat-release
uname -sr

设置主机名及主机名解析

#各节点，分别进行设置
hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2

cat >> /etc/hosts << EOF
127.0.0.1   $(hostname)
192.168.8.11 k8s-master
192.168.8.22 k8s-node1
192.168.8.33 k8s-node2
EOF

时间同步

ntpdate time.windows.com
#命令不存在则执行
yum install ntpdate -y

防火墙关闭，仅开发环境

systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld

关闭SELinux

#查看是否开启
getenforce
#关闭，需要重启主机
sed -i 's/enforcing/disabled/' /etc/selinux/config
#再次查看

关闭swap

#需要重启
sed -ri 's/.*swap.*/#&/' /etc/fstab

#查看，swap是否为0
free -m

iptables配置

#将桥接的 IPv4 流量传递到 iptables 的链
# 可能没有，追加
echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.all.forwarding = 1"  >> /etc/sysctl.conf

modprobe br_netfilter
sysctl -p

开启ipvs

yum -y install ipset ipvsadm
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

部署docker

三台虚拟机均需部署

#安装docker
##卸载旧版本
sudo yum remove docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-engine

yum -y install gcc
yum -y install gcc-c++
yum -y install yum-utils
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum list docker-ce --showduplicates | sort -r
yum -y install docker-ce-3:20.10.8-3.el7.x86_64 docker-ce-cli-1:20.10.8-3.el7.x86_64 containerd.io
# 启动 Docker 
systemctl start docker
# 开启自动启动
systemctl enable docker
#查看是否安装成功
docker version

#镜像加速配置
sudo mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "exec-opts": ["native.cgroupdriver=systemd"],	
  "registry-mirrors": [
    "https://du3ia00u.mirror.aliyuncs.com",
    "https://docker.m.daocloud.io",
    "https://hub-mirror.c.163.com",
    "https://mirror.baidubce.com",
    "https://docker.nju.edu.cn",
    "https://docker.mirrors.sjtug.sjtu.edu.cn"
  ],
  "live-restore": true,
  "log-driver":"json-file",
  "log-opts": {"max-size":"500m", "max-file":"3"},
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 5,
  "storage-driver": "overlay2"
}
EOF

#重启docker
sudo systemctl daemon-reload
sudo systemctl restart docker

k8s安装

k8s镜像获取

#添加阿里云的 Kubernetes 的 YUM 源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安装 kubelet 、kubeadm 和  kubectl
yum install -y kubelet-1.21.10 kubeadm-1.21.10 kubectl-1.21.10

vim /etc/sysconfig/kubelet
# 修改
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
KUBE_PROXY_MODE="ipvs"
#开机自启动
systemctl enable kubelet


kubeadm config images list
#有大佬已经将墙外的 Docker 镜像同步到 DockerHub 中，
#地址在这里：https://github.com/x-mirrors/gcr.io ，大家可以去看看。
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.21.10
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.21.10
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.21.10
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.21.10
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.4.1
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.13-0
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.0
#给 coredns 镜像重新打 tag
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.0 registry.cn-hangzhou.aliyuncs.com/google_containers/coredns/coredns:v1.8.0

部署master节点

在 192.168.8.11 机器上部署 Kubernetes 的 Master 节点

# 由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里需要指定阿里云镜像仓库地址
kubeadm init \
  --apiserver-advertise-address=192.168.8.11 \
  --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers \
  --kubernetes-version=v1.21.10 \
  --service-cidr=10.96.0.0/16 \
  --pod-network-cidr=10.244.0.0/16
  
#注意：
#● apiserver-advertise-address 一定要是主机的 IP 地址。 
#● apiserver-advertise-address 、service-cidr 和 pod-network-cidr 不能在同一个网络范围内。
#● 不要使用 172.17.0.1/16 网段范围，因为这是 Docker 默认使用的。

执行成功的完整日志内容：

[root@k8s-master ~]# kubeadm init \
>   --apiserver-advertise-address=192.168.8.11 \
>   --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers \
>   --kubernetes-version=v1.21.10 \
>   --service-cidr=10.96.0.0/16 \
>   --pod-network-cidr=10.244.0.0/16
[init] Using Kubernetes version: v1.21.10
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.8.11]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master localhost] and IPs [192.168.8.11 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master localhost] and IPs [192.168.8.11 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 6.501963 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.21" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node k8s-master as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: hrshel.j815bn8ato6fea8u
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.8.11:6443 --token hrshel.j815bn8ato6fea8u \
        --discovery-token-ca-cert-hash sha256:ba5ee81176dca1b21bdbdae9488dc4699131e4bcec5e860556e7b4c0ff7644c4 

根据日志提示，执行脚本

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 如果是 root 用户，还可以执行如下命令
export KUBECONFIG=/etc/kubernetes/admin.conf

#默认的 token 有效期为 24 小时，当过期之后，该 token 就不能用了，这时可以使用如下的命令创建 token 
kubeadm token create --print-join-command
# 生成一个永不过期的token
kubeadm token create --ttl 0 --print-join-command

部署node节点

#根据上面的日志提示，在两个节点执行如下命令
kubeadm join 192.168.8.11:6443 --token 2y6m6a.1qeo40nz4oenbcms --discovery-token-ca-cert-hash sha256:ba5ee81176dca1b21bdbdae9488dc4699131e4bcec5e860556e7b4c0ff7644c4

此时查看节点状态，都是not ready状态
kubectl get nodes

部署网络插件

#在master节点执行
kubectl apply -f https://projectcalico.docs.tigera.io/v3.19/manifests/calico.yaml

#查看部署 CNI 网络插件进度
kubectl get pods -n kube-system
#或
watch kubectl get pods -n kube-system

再次查看节点状态

kubectl edit cm kube-proxy -n kube-system
#修改mode为"ipvs"模式

#删除 kube-proxy ，让 Kubernetes 集群自动创建新的 kube-proxy ：
kubectl delete pod -l k8s-app=kube-proxy -n kube-system

允许Node节点使用kubectl

让 Node 节点也能使用 kubectl 命令
默认情况下，只有 Master 节点才有 kubectl 命令，但是有些时候我们也希望在 Node 节点上执行 kubectl 命令
# node节点执行 192.168.8.22 和 192.168.8.33
mkdir -pv ~/.kube
touch ~/.kube/config

# 主节点执行
# 192.168.8.11
scp /etc/kubernetes/admin.conf [email protected]:~/.kube/config
# 192.168.8.11
scp /etc/kubernetes/admin.conf [email protected]:~/.kube/config

测试部署nginx

#部署 Nginx 
kubectl create deployment nginx --image=nginx:1.14-alpine
#暴露端口
kubectl expose deployment nginx --port=80 --type=NodePort
#查看服务状态
kubectl get pods,svc

部署成功

k8s命令，自动补全配置

# 安装 
yum -y install bash-completion
# 自动补全 
echo 'source <(kubectl completion bash)' >>~/.bashrc 
kubectl completion bash >/etc/bash_completion.d/kubectl
# 全局
kubectl completion bash | sudo tee /etc/bash_completion.d/kubectl > /dev/null
source /usr/share/bash-completion/bash_completion

kuboard安装

入门学习，官方推荐使用内建用户库方式。
在主节点运行即可

sudo docker run -d \
  --restart=unless-stopped \
  --name=kuboard \
  -p 80:80/tcp \
  -p 10081:10081/tcp \
  -e KUBOARD_ENDPOINT="http://k8s-master:80" \
  -e KUBOARD_AGENT_SERVER_TCP_PORT="10081" \
  -v /root/kuboard-data:/data \
  eipwork/kuboard:v3
  
  # 也可以使用镜像 swr.cn-east-2.myhuaweicloud.com/kuboard/kuboard:v3 ，可以更快地完成镜像下载。
  # 请不要使用 127.0.0.1 或者 localhost 作为内网 IP \
  # Kuboard 不需要和 K8S 在同一个网段，Kuboard Agent 甚至可以通过代理访问 Kuboard Server \

默认

• 用户名： admin
• 密 码： Kuboard123

访问，添加已有集群，根据提示操作即可，相当人性化。

在名称空间中，可以查看到之前创建的nginx

需要在K8s中安装metrics-server，否则无法查看计算资源信息
（具体操作，见许大仙笔记中的k8s生态链路整合篇）
安装完毕后，可以看到集群的cpu、内存信息