文档将介绍如何使用二进制部署Kubernetes v1.16高可用集群，而不是使用自动化部署(kubeadm)集群。在部署过程中，将详细列出各个组件启动参数，以及相关配置说明。部署完成后，将理解k8s各个组件的交互原理，并且可以快速解决实际问题。

 

* 环境及组件版本

    Centos7.2

    Kubernetes v1.16.10

    Docker 18.09 .01(需要和k8s版本匹配)

    Etcd 3.3.13

    Flanneld 0.12.0

 

  • 一、 环境准备

  1. 服务器准备

     master节点

10.13.33.29  master-01
10.13.33.40  master-02
10.13.33.38  master-03

     node节点

10.13.33.31  node-01


  1. 修改主机名

     为对应服务器设置永久主机名

     

cat >> /etc/hosts < 
  
     
   
  1. 更新PATH变量
    2.  
  
 
  
           k8s所有文件和配置都会存储在/opt/k8s/ 中,运行以下命令，配置环境变量
 
  
echo 'PATH=/opt/k8s/bin:$PATH' >>/etc/profile    ##将k8s命令添加到环境变量中方便调用
source  /etc/profil
 
  
     
   
  •      二、服务器优化
    •  
  
 
  

 
  
          所有节点配置
 
  
          如果不能访问互联网请使用以下代理配置
 
  
echo "proxy=http://ip：port" >>/etc/yum.conf
export http_proxy="http://ip：port"
 
  
        1、 创建相关目录
 
  
mkdir -p  /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
cd /opt/k8s/work
wget  wget https://github.com/kubernetes/kubernetes/archive/v1.16.10.tar.gz ##下载k8s
 
  
        2、 服务器时间写入硬件
 
  
timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 0
 
  
     
   
  1.     3、配置时钟服务器地址
    2.  
  
 
  
echo "
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst

# Ignore stratum in source selection.
stratumweight 0

# Record the rate at which the system clock gains/losses time.
driftfile /var/lib/chrony/drift

# Enable kernel RTC synchronization.
rtcsync

# In first three updates step the system clock instead of slew
# if the adjustment is larger than 10 seconds.
makestep 10 3

# Allow NTP client access from local network.
#allow 192.168/16

# Listen for commands only on localhost.
bindcmdaddress 127.0.0.1
bindcmdaddress ::1

# Serve time even if not synchronized to any NTP server.
#local stratum 10

keyfile /etc/chrony.keys

# Specify the key used as password for chronyc.
commandkey 1

# Generate command key if missing.
generatecommandkey

# Disable logging of client accesses.
noclientlog

# Send a message to syslog if a clock adjustment is larger than 0.5 seconds.
logchange 0.5

logdir /var/log/chrony
#log measurements statistics tracking
server 时钟ip1 iburst prefer
server 时钟ip2 iburst
server 127.127.1.0
allow 0/0
local stratum 11
" > /etc/chrony.conf

systemctl restart chronyd                    ##重启时钟应用配置
systemctl status chronyd                     ##查看启动状态
systemctl enable chronyd                     ##开机自动启动
firewall-cmd --add-service=ntp --permanent   ##放开对应的防火墙
firewall-cmd --add-service=ntp               ##放开对应的防火墙
systemctl restart rsyslog 
systemctl restart crond
timedatectl status
chronyc sources -v                           ##验证时钟地址
 
  
             4、关闭其他非必要服务
 
  
systemctl stop postfix && systemctl disable postfix
 
  
              5、 持久化保存日志的目录
 
  
mkdir /var/log/journal  #持久化保存日志的目录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf << EOF
[Journal]
Storage=persistent
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
SystemMaxUse=10G
SystemMaxFileSize=200M
MaxRetentionSec=2week
ForwardToSyslog=no
EOF
systemctl restart systemd-journald
 
  
             6、关闭防火墙
 
  
systemctl stop firewalld
systemctl disable firewalld
 
  
             7、优化iptables
 
  
iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat
iptables -P FORWARD ACCEPT
 
  
             8、关闭swap及SELINUX
 
  
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
#如果开启了swap分区，kubelet会启动失败(可以通过设置参数——-fail-swap-on设置为false)
 
  
             9、优化内核等
 
  
echo "
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
net.ipv4.conf.all.forwarding=1
net.ipv4.neigh.default.gc_thresh1=4096
net.ipv4.neigh.default.gc_thresh2=6144
net.ipv4.neigh.default.gc_thresh3=8192
net.ipv4.neigh.default.gc_interval=60
net.ipv4.neigh.default.gc_stale_time=120
# 参考 https://github.com/prometheus/node_exporter#disabled-by-default
kernel.perf_event_paranoid=-1
#sysctls for k8s node config
net.ipv4.tcp_slow_start_after_idle=0
net.core.rmem_max=16777216
fs.inotify.max_user_watches=524288
kernel.softlockup_all_cpu_backtrace=1
kernel.softlockup_panic=0
kernel.watchdog_thresh=30
fs.file-max=2097152
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
vm.max_map_count=262144
fs.may_detach_mounts=1
net.core.netdev_max_backlog=16384
net.ipv4.tcp_wmem=4096 12582912 16777216
net.core.wmem_max=16777216
net.core.somaxconn=32768
net.ipv4.ip_forward=1
net.ipv4.tcp_max_syn_backlog=8096
net.ipv4.tcp_rmem=4096 12582912 16777216
net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
net.ipv6.conf.lo.disable_ipv6=1
kernel.yama.ptrace_scope=0
vm.swappiness=0
# 可以控制core文件的文件名中是否添加pid作为扩展。
kernel.core_uses_pid=1
# Do not accept source routing
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_source_route=0
# Promote secondary addresses when the primary address is removed
net.ipv4.conf.default.promote_secondaries=1
net.ipv4.conf.all.promote_secondaries=1
# Enable hard and soft link protection
fs.protected_hardlinks=1
fs.protected_symlinks=1
# 源路由验证
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets=5000
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_synack_retries=2
kernel.sysrq=1
" >> /etc/sysctl.conf

sysctl -p

###nofile
cat >> /etc/security/limits.conf < 
  
            10、更新内核版本
 
  
echo  Kernel_Version
uname -srm
export Kernel_Version=4.18.9-1
wget  http://mirror.rc.usf.edu/compute_lock/elrepo/kernel/el7/x86_64/RPMS/kernel-ml{,-devel}-${Kernel_Version}.el7.elrepo.x86_64.rpm
yum localinstall -y kernel-ml*
grub2-set-default 'CentOS Linux (4.18.9-1.el7.elrepo.x86_64) 7 (Core)'
shutdown -r -t 0                      ##如果没有自行重启，进行手动重启
uname -srm
 
  
             11、使用systemd-modules-load加载内核模块
 
  
touch /etc/sysconfig/modules/ipvs.modules

cat > /etc/sysconfig/modules/ipvs.modules << EOF 
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
 
  
     
   
  •       三、部署安装k8s组件
    •  
  
 
  
            1、制作证书
 
  
##安装cfssl工作集
mkdir -p /opt/k8s/cert && cd /opt/k8s
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
mv cfssl_linux-amd64 /opt/k8s/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
mv cfssljson_linux-amd64 /opt/k8s/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo
chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH
 
  
##创建更证书及配置文件
cd  /opt/k8s
mv ~/cfssl* ./
mv cfssl_linux-amd64 /opt/k8s/bin/cfssl
mv cfssljson_linux-amd64 /opt/k8s/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /opt/k8s/bin/cfssl-certinfo
chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH

##master1
cd /opt/k8s/work
cat > ca-config.json < 
  
##创建证书签名请求文件
cat > ca-csr.json < 
  
##生成CA证书和私钥
cd /opt/k8s/work
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*
##将证书分发到各个节点
 
  
##创建flannel证书
cat > flanneld-csr.json < 
  
     
   
  1.         2、部署flannel
    2.  
  
 
  
###安装flannel

cd /opt/k8s/work/
mkdir flannel
cp -f ~/k8s/ca*.pem /etc/kubernetes/cert/
wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz
tar -xzvf flannel-v0.12.0-linux-amd64.tar.gz -C flannel
cp flannel/{flanneld,mk-docker-opts.sh} /opt/k8s/bin/
chmod +x /opt/k8s/bin/*

##创建flanneld.service
touch /etc/systemd/system/flanneld.service
cat > /etc/systemd/system/flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
ExecStart=/opt/k8s/bin/flanneld \\
  -etcd-cafile=/etc/kubernetes/cert/ca.pem \\
  -etcd-certfile=/etc/flanneld/cert/flanneld.pem \\
  -etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\
  -etcd-endpoints=https://10.13.33.29:2379,https://10.13.33.38:2379,https://10.13.33.40:2379 \\
  -etcd-prefix=/kubernetes/network \\
  -iface=eth0 \\
  -ip-masq
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF

##启动flannel
systemctl daemon-reload && systemctl enable flanneld && systemctl restart flanneld || systemctl status flanneld|grep Active
systemctl status flanneld             ##查看flanneld运行状态，依赖docker可能会启动失败，后面安装docker后会启动成功
ip addr show|grep flannel.1
 
  
            3、安装docker
 
  
##更新yum源
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/backup/
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum update -y
yum install -y yum-utils device-mapper-persistent-data lvm2 nfs-utils                              ##安装依赖包
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo     ##使用阿里云的docker镜像站
yum makecache fast

##安装指定版本的docker
yum -y install docker-ce-18.09.0-3.el7 docker-ce-cli-18.09.0-3.el7 containerd.io

##优化docker.service
cat > /usr/lib/systemd/system/docker.service < /etc/docker/daemon.json < 
  
     
   

    2.