使用confd与nginx 实现kubernetes master节点高可用

下载confd 二进制文件

# 创建目录方便存放文件
mkdir confd
# 进入新创建的目录
cd confd
# 下载 confd
wget https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64
# 重命名 
mv confd-0.16.0-linux-amd64  confd
# 给confd 可执行权限
chmod +x confd

生成confd 配置

# 创建confd 配置目录
mkdir -p ./conf.d
# 创建模版存放目录
mkdir -p ./templates
# 生成confd 配置文件
cat << EOF | tee ./conf.d/nginx.toml
[template]
src = "nginx.tmpl"
dest = "/etc/nginx/nginx.conf"
keys = [
    "CP_HOSTS",
]
EOF
# 生成模版文件
cat << EOF | tee ./templates/nginx.tmpl
error_log stderr notice;

worker_processes auto;
events {
  multi_accept on;
  use epoll;
  worker_connections 4096;
}

stream {
        upstream kube_apiserver {
            {{ \$servers := split (getenv "CP_HOSTS") "," }}{{range \$servers}}
            server {{.}}:6443;
            {{end}}
        }

        server {
            listen        6443;
            proxy_pass    kube_apiserver;
            proxy_timeout 30;
            proxy_connect_timeout 2s;

        }

}
EOF
# 生成启动文件
cat << EOF | tee ./nginx-proxy
#!/bin/sh

# Run confd
confd -onetime -backend env

# Start nginx
nginx -g 'daemon off;'
EOF
# 给启动文件执行权限
chmod +x ./nginx-proxy

Dockerfile

vim  Dockerfile
# 基础镜像
FROM alpine

# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "[email protected]"

# 修改源
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories

# 安装ca 证书
RUN apk update && \
    apk add --no-cache ca-certificates 

# 设置环境变量

ENV NGINX_VERSION 1.17.9

ENV OPENSSL_VERSION 1.1.1e

# 编译安装NGINX

WORKDIR /tmp

RUN NGINX_CONFIG="\
      --prefix=/etc/nginx \
      --sbin-path=/usr/sbin/nginx \
      --conf-path=/etc/nginx/nginx.conf \
      --error-log-path=/var/log/nginx/error.log \
      --http-log-path=/var/log/nginx/access.log \
      --pid-path=/var/run/nginx.pid \
      --lock-path=/var/run/nginx.lock \
      --http-client-body-temp-path=/var/cache/nginx/client_temp \
      --http-proxy-temp-path=/var/cache/nginx/proxy_temp \
      --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
      --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
      --http-scgi-temp-path=/var/cache/nginx/scgi_temp \
      --with-pcre \
      --user=nginx \
      --group=nginx \
      --with-compat \
      --with-file-aio \
      --with-threads \
      --with-http_addition_module \
      --with-http_auth_request_module \
      --with-http_dav_module \
      --with-http_flv_module \
      --with-http_gunzip_module \
      --with-http_gzip_static_module \
      --with-http_mp4_module \
      --with-http_random_index_module \
      --with-http_realip_module \
      --with-http_secure_link_module \
      --with-http_slice_module \
      --with-http_ssl_module \
      --with-http_stub_status_module \
      --with-http_sub_module \
      --with-http_v2_module \
      --with-ipv6 \
      --with-openssl=../openssl-$OPENSSL_VERSION \
      --with-openssl-opt=enable-tls1_3 \
      --with-mail \
      --with-mail_ssl_module \
      --with-stream \
      --with-stream_realip_module \
      --with-stream_ssl_module \
      --with-stream_ssl_preread_module \
      --with-ld-opt=-Wl,--as-needed \
     " \
     && addgroup -S nginx \
     && adduser -D -S -h /www -s /sbin/nologin -G nginx nginx \
     && apk  add  --no-cache --virtual .build-deps \
        gcc \
        libc-dev \
        make \
        pcre-dev \
        zlib-dev \
        linux-headers \
        curl \
        gnupg \
        libxslt-dev \
        gd-dev \
        geoip-dev \
        libstdc++ wget \
        libjpeg  \
        libpng \
        libpng-dev \
        freetype \
        freetype-dev \
        libxml2 \
        libxml2-dev \
        curl-dev \
        libmcrypt \
        libmcrypt-dev \
        autoconf \
        libjpeg-turbo-dev \
        libmemcached \
        libmemcached-dev \
        gettext \
        gettext-dev \
        libzip \
        git \
        libzip-dev \
        && curl -fSL  https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl-$OPENSSL_VERSION.tar.gz \
        && curl -fSL https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -o /tmp/nginx-$NGINX_VERSION.tar.gz \
        && cd /tmp \
        && tar -xzf openssl-$OPENSSL_VERSION.tar.gz \
        && tar -xzf nginx-$NGINX_VERSION.tar.gz \
        && cd  /tmp/nginx-$NGINX_VERSION \
        && ./configure $NGINX_CONFIG \
        && make -j$(getconf _NPROCESSORS_ONLN) \
        && make install

# 构建confd nginx 镜像

FROM alpine 
# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "[email protected]"

# 修改源
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories

# 安装ca 证书
RUN apk update && \
    apk add --no-cache ca-certificates 

# 设置环境变量

ENV NGINX_VERSION 1.17.9

ENV OPENSSL_VERSION 1.1.1e

RUN  mkdir -p /var/lib/nginx/cache \
     && apk add  --no-cache  \ 
           curl \
           wget \
           pcre \
          && addgroup -S nginx \
          && adduser -D -S -h /var/lib/nginx -s /sbin/nologin -G nginx nginx \
        && chown -R nginx:nginx /var/lib/nginx \
        && mkdir -p /var/log/nginx \
        && rm -rf /var/cache/apk/* \
        && mkdir -p /etc/confd \
        && mkdir -p /var/cache/nginx/client_temp
#COPY 编译结果  

COPY --from=0  /usr/sbin/nginx /usr/sbin/nginx
COPY --from=0  /etc/nginx  /etc/nginx  
ADD confd  /usr/sbin/confd
ADD conf.d /etc/confd/conf.d 
ADD templates /etc/confd/templates
ADD nginx-proxy /usr/bin/nginx-proxy

STOPSIGNAL SIGTERM

ENTRYPOINT ["/usr/bin/nginx-proxy"]

生成镜像

[root@nginx-1 confd]# tree
.
|-- Dockerfile
|-- conf.d
|   `-- nginx.toml
|-- confd
|-- nginx-proxy
`-- templates
    `-- nginx.tmpl

2 directories, 5 files
# 生成镜像
docker build -t ha-tools:v1.17.9 .     # 镜像名字自己修改 我这里以ng 版本为tag
# 给进行打新tag
docker tag ha-tools:v1.17.9  juestnow/ha-tools:v1.17.9
# 上传镜像
docker push  juestnow/ha-tools:v1.17.9

测试生成的镜像

# 单个IP 
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175" juestnow/ha-tools:v1.17.9s CP_HOSTS=192.168.2.175
# 多个IP 
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 进去容器查看是否正常
docker ps
docker exec -ti 27733e5f9a97 /bin/sh
/ # ps -ef
PID   USER     TIME  COMMAND
    1 root      0:00 {nginx-proxy} /bin/sh /usr/bin/nginx-proxy CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
   12 root      0:00 nginx: master process nginx -g daemon off;
   13 nginx     0:00 nginx: worker process
   14 nginx     0:00 nginx: worker process
   15 nginx     0:00 nginx: worker process
   16 nginx     0:00 nginx: worker process
   17 root      0:00 /bin/sh
   22 root      0:00 ps -ef
     # 查看端口监听
     / # netstat -tnlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:6443            0.0.0.0:*               LISTEN      12/nginx: master pr
# 验证访问
/ # curl -k https://127.0.0.1:6443
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {

  },
  "status": "Failure",
  "message": "Unauthorized",
  "reason": "Unauthorized",
  "code": 401
}/ #
代理正常有数据返回

k8s 使用 ha-tools

# kube-apiserver 节点不部署 ha-tools 只是node 节点部署
# 二进制部署kube-apiserver 证书签名时加上127.0.0.1 这个IP 以后整个集群访问都走127.0.0.1 这个IP+端口 同时kube-apiserver 改成0.0.0.0如果不修改master 安装kubelet 的时候记得修改IP 
# kubeadm 安装时 请加入apiserver-cert-extra-sans=127.0.0.1 这样才能127.0.0.1 访问不然会一致报错
# 每个node 节点运行
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 还可以放到kubelet manifests 目录
[root@nginx-1 manifests]# cat ha-tools.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: ha-tools
    tier: control-plane
  name: ha-tools
  namespace: kube-system
spec:
  containers:
  - args:
    - "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177"  
    image: juestnow/ha-tools:v1.17.9
    imagePullPolicy: IfNotPresent
    name: ha-tools
    env:
    - name: CP_HOSTS
      value: "192.168.2.175,192.168.2.176,192.168.2.177"
  hostNetwork: true
  priorityClassName: system-cluster-critical
status: {}
# 二进制方式部署推荐使用以上的方式
[root@localhost ~]# kubectl get pod -A | grep ha-tools
kube-system   ha-tools-nginx-1                                1/1     Running   0          14h

# kubeadm 方式部署高可用修改kube-proxy 让它连接127.0.0.1
kubectl -n kube-system edit configmaps kube-proxy
# 二进制部署直接在 kubeconfig 添加就可以