搭建K8S-web-MySQL-keepalived 负载均衡集群项目

搭建K8S-web-MySQL-keepalived 负载均衡集群项目

总体架构图

架构图说明：
keepalivd部分：k8s内部的ip无法实现外部访问，这样用户是无法使用的，所以用k8s节点的ip段做VIP让外网可以用唯一的一个ip访问集群。双节点保证了可以用性。
web部分：django做的web集群，又绑定了svc的NodePORT模式，此时访问k8s集群所有的节点都可以访问到的，但是用户不能不断的变化访问IP，所以又交给keepalived去实现单一IP访问。web架构也用deployment模式管理快速升级，变更版本。（注意svc的rr轮训机制，并不保留web用户的seesion需要配合开发将seesion保存到数据库或文件中，实现会话统一）。
Mysql部分：同样使用svc+deployment的方式管理mysql镜像，方便版本变更，单点故障。但是只用了1个副本管理的方式，因为多副本会造成数据不一致。同时使用pv+pvc的方式持久化存储。

安装MySQL

下载MySQL镜像（网速不好，提前下载，不容易报错）

docker pull mysql:5.7.38
docker tag mysql:5.7.38  c104.cheng.com/cheng/mysql:5.7.38

#推送到自己的私有仓库，因为默认k8s集群，不会所有 节点都有本地镜像的，如果不放自己的私有仓库，都是走公网仓库，肯能拉取太慢而失败！

docker login https://c104.cheng.com -u admin -p Harbor12345

#如果没有私有仓库参考我的搭建Harbor文章

docker push  c104.cheng.com/cheng/mysql:5.7.38

创建一个Mysql的deployment模板

创建一个Mysql的deployment模板，然后去修改它，不可能手写那么多基础标签，而且容易出错。

kubectl create deployment mysql5738 --image=c104.cheng.com/cheng/mysql:5.7.38

#此时启动会有问题，因为默认mysql启动是要参数的！所以导出yaml文件，增加标签：ports、env。
#导出json、yaml皆可以。（现在只有一个deploy，可以不写名字）

kubectl get deployment  mysql5738 -o json > mysql5738.json   

#修改完成后，删除重新创建。（deployment的名字不要有 点 不然 dns解析会有报错）

kubectl delete deployment  mysql5738
kubectl apply  deployment  mysql5738 -f   mysql5738.json 

#查看创建的deploy、rs、pod，只有deploy是自己定义的名字，其他名字都是随机加了后缀！我们创建deploy的时候，本质创建了三种资源，切记，也是创建pod的常用方式。

kubectl get deploy
kubectl get rs
kubectl get pod

#注意：
#pod漂移：此特性会造成单个pod的业务会中断。切记！！！！！
#mysql镜像的数据，没有映射到本地，就会缓存在 镜像的缓存层！此时删除pod，自动创建，但是数据丢失！！！
#节点漂移默认5分钟，期间get 查看都是running的，因为kubelet还没上报到api
#mysql pod docker 内存少于1G会出现，Back-off restarting failed container ，重启异常
#如果节点中有正常启动pod后，那些Back-off 的会变成 Evicted 状态 #这种僵尸pod，本质已经死了。因为资源不够导致的。可以直接删除

为mysql的pod绑定svc

此时svc的名字和deployment 是一样的。注意理解。

kubectl expose deployment mysql5738 --port=3306 --target-port=3306 --protocol=TCP --type=ClusterIP

#查看ClusterIP，为了以后连接mysql，如：

kubectl get svc

搭建nfs部署pv

yum install -y  nfs-utils rpcbind
mkdir /root/data  &&  echo "/root/data/ *(insecure,rw,sync,no_root_squash)" > /etc/exports
chown -R nfsnobody.nfsnobody /root/data
systemctl start rpcbind  &&  systemctl start nfs-server && systemctl enable rpcbind  &&  systemctl enable nfs-server
#验证
showmount -e localhost
rpcinfo -p localhost
cd /root/data
mkdir mysqletc mysqldata
chown -R nfsnobody.nfsnobody ./*
#所有k8s节点
yum install -y  nfs-utils rpcbind
systemctl start rpcbind
systemctl enable  rpcbind
systemctl status  rpcbind

#直接使用nfs，挂载到pod中：略 （注意挂载的容器目录，不要是有内容的，会清空的。）
#或者创建pv的方式：优点，可以控制文件大小，传输速度等！
#pvc会根据pv的大小，精准匹配绑定；如何没合适的，找稍微大的pv
如果绑定后单独删除一方，另一方不会释放，等待另一方的创建，后再连接。如果更换双方，需要删除自己的信息（yaml文件中）
#还有自动分配pv的类型StorageClass；pvc找不到合适的pv，就会自动创建合适的pv
#先创建pv 再创建pvc有合适的自动绑定

#kubelet 异常报错，因为kubelet开机会统计cpu等资源信息。
编辑 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf 文件，并添加下面配置：

CPUAccounting=true
MemoryAccounting=true

附：涉及的yaml文件清单

apiVersion: v1
items:
- apiVersion: extensions/v1beta1
  kind: Deployment
  metadata:
    annotations:
      deployment.kubernetes.io/revision: "1"
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"extensions/v1beta1","kind":"Deployment","metadata":{"annotations":{"deployment.kubernetes.io/revision":"1"},"creationTimestamp":"2022-06-15T21:52:06Z","generation":1,"labels":{"app":"mysql5738"},"name":"mysql5738","namespace":"default","resourceVersion":"224507","selfLink":"/apis/extensions/v1beta1/namespaces/default/deployments/mysql5738","uid":"8e0a6f8e-6e7d-4735-bcb7-13ff5f7d2ded"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"mysql5738"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"mysql5738"}},"spec":{"containers":[{"env":[{"name":"MYSQL_ROOT_PASSWORD","value":"123456"}],"image":"c104.cheng.com/cheng/mysql:5.7.38","imagePullPolicy":"IfNotPresent","name":"mysql","ports":[{"containerPort":3306,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/var/lib/mysql","name":"mysqldatapvc"}]}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30,"volumes":[{"name":"mysqldatapvc","persistentVolumeClaim":{"claimName":"mysql5738pvc"}}]}}},"status":{"conditions":[{"lastTransitionTime":"2022-06-15T21:52:06Z","lastUpdateTime":"2022-06-15T21:52:06Z","message":"Deployment does not have minimum availability.","reason":"MinimumReplicasUnavailable","status":"False","type":"Available"},{"lastTransitionTime":"2022-06-15T21:52:06Z","lastUpdateTime":"2022-06-15T21:52:06Z","message":"ReplicaSet \"mysql5738-59cd758d7b\" is progressing.","reason":"ReplicaSetUpdated","status":"True","type":"Progressing"}],"observedGeneration":1,"replicas":1,"unavailableReplicas":1,"updatedReplicas":1}}
    creationTimestamp: "2022-06-16T11:31:44Z"
    generation: 1
    labels:
      app: mysql5738
    name: mysql5738
    namespace: default
    resourceVersion: "297465"
    selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/mysql5738
    uid: 1df44519-4ecf-4bcb-8428-42d386b35ccf
  spec:
    progressDeadlineSeconds: 600
    replicas: 1
    revisionHistoryLimit: 10
    selector:
      matchLabels:
        app: mysql5738
    strategy:
      rollingUpdate:
        maxSurge: 25%
        maxUnavailable: 25%
      type: RollingUpdate
    template:
      metadata:
        creationTimestamp: null
        labels:
          app: mysql5738
      spec:
        containers:
        - env:
          - name: MYSQL_ROOT_PASSWORD
            value: "123456"
          image: c104.cheng.com/cheng/mysql:5.7.38
          imagePullPolicy: IfNotPresent
          name: mysql
          ports:
          - containerPort: 3306
            protocol: TCP
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
          - mountPath: /var/lib/mysql
            name: mysqldatapvc
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        schedulerName: default-scheduler
        securityContext: {}
        terminationGracePeriodSeconds: 30
        volumes:
        - name: mysqldatapvc
          persistentVolumeClaim:
            claimName: mysql5738pvc
  status:
    availableReplicas: 1
    conditions:
    - lastTransitionTime: "2022-06-16T11:31:45Z"
      lastUpdateTime: "2022-06-16T11:31:45Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
    - lastTransitionTime: "2022-06-16T11:31:44Z"
      lastUpdateTime: "2022-06-16T11:31:45Z"
      message: ReplicaSet "mysql5738-f64f7f77f" has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    observedGeneration: 1
    readyReplicas: 1
    replicas: 1
    updatedReplicas: 1
- apiVersion: v1
  kind: PersistentVolume
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"v1","kind":"PersistentVolume","metadata":{"annotations":{},"name":"mysql5738pv"},"spec":{"accessModes":["ReadWriteMany"],"capacity":{"storage":"10Gi"},"nfs":{"path":"/root/data/mysqldatapv","server":"192.168.18.104"}}}
      pv.kubernetes.io/bound-by-controller: "yes"
    creationTimestamp: "2022-06-16T11:17:09Z"
    finalizers:
    - kubernetes.io/pv-protection
    name: mysql5738pv
    resourceVersion: "296682"
    selfLink: /api/v1/persistentvolumes/mysql5738pv
    uid: efb652c2-d353-424c-87e3-73ea3b87af70
  spec:
    accessModes:
    - ReadWriteMany
    capacity:
      storage: 10Gi
    claimRef:
      apiVersion: v1
      kind: PersistentVolumeClaim
      name: mysql5738pvc
      namespace: default
      resourceVersion: "296337"
      uid: 3df9df5b-4f0b-4aeb-ad61-5d495f3ec451
    nfs:
      path: /root/data/mysqldatapv
      server: 192.168.18.104
    persistentVolumeReclaimPolicy: Retain
    volumeMode: Filesystem
  status:
    phase: Bound
- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"v1","kind":"PersistentVolumeClaim","metadata":{"annotations":{},"name":"mysql5738pvc","namespace":"default"},"spec":{"accessModes":["ReadWriteMany"],"resources":{"requests":{"storage":"200Mi"}}}}
      pv.kubernetes.io/bind-completed: "yes"
      pv.kubernetes.io/bound-by-controller: "yes"
    creationTimestamp: "2022-06-16T11:19:21Z"
    finalizers:
    - kubernetes.io/pvc-protection
    name: mysql5738pvc
    namespace: default
    resourceVersion: "296684"
    selfLink: /api/v1/namespaces/default/persistentvolumeclaims/mysql5738pvc
    uid: 3df9df5b-4f0b-4aeb-ad61-5d495f3ec451
  spec:
    accessModes:
    - ReadWriteMany
    resources:
      requests:
        storage: 200Mi
    volumeMode: Filesystem
    volumeName: mysql5738pv
  status:
    accessModes:
    - ReadWriteMany
    capacity:
      storage: 10Gi
    phase: Bound
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2022-06-13T23:52:28Z"
    labels:
      component: apiserver
      provider: kubernetes
    name: kubernetes
    namespace: default
    resourceVersion: "151"
    selfLink: /api/v1/namespaces/default/services/kubernetes
    uid: 4106c941-92eb-4ee6-9b71-2dd06ca8b473
  spec:
    clusterIP: 10.96.0.1
    ports:
    - name: https
      port: 443
      protocol: TCP
      targetPort: 6443
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2022-06-15T22:26:52Z"
    labels:
      app: mysql5738
    name: mysql5738
    namespace: default
    resourceVersion: "227501"
    selfLink: /api/v1/namespaces/default/services/mysql5738
    uid: 9d63ff56-31cb-449c-9954-120ea93181e8
  spec:
    clusterIP: 10.102.125.157
    ports:
    - port: 3306
      protocol: TCP
      targetPort: 3306
    selector:
      app: mysql5738
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

WEB程序安装

docker安装web程序

测试系统正常：创建数据库，拉取镜像，启动

CREATE DATABASE `djangoblog` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci */;
docker pull liangliangyy/djangoblog:latest
docker run -d  -p 80:8000 -e DJANGO_MYSQL_HOST=10.102.125.157 -e DJANGO_MYSQL_PASSWORD=123456 -e DJANGO_MYSQL_USER=root -e DJANGO_MYSQL_DATABASE=djangoblog --name djangoblog liangliangyy/djangoblog:latest

#创建web网站用户

docker exec -it djangoblog python /code/djangoblog/manage.py createsuperuser

#验证

#http://192.168.18.101/ 

改web的docker为k8s部署

#测试可以正常访问，改用k8s部署。

docker stop acb44d26573e
docker rm acb44d26573e

#首先创建deployment模板，然后删除它，修改yaml文件，增加启动参数;这样可以快速得到yaml文件的参数！！！

kubectl create deployment djangoblog --image=c104.cheng.com/cheng/djangoblog:v1.1.1
kubectl get deploy djangoblog -o yaml >djangoblog111.yaml
kubectl delete  -f  djangoblog111.yaml
#修改env和ports参数后，重新创建deployment
kubectl apply -f djangoblog111.yaml
#为pod绑定，此时svc的名字和deployment
kubectl expose deployment djangoblog  --target-port=8000 --protocol=TCP --type=NodePort

##查看绑定情况：

kubectl get svc -o wide

##扩容到3个pod，创建负载均衡的节点

kubectl scale deployment djangoblog --replicas=3

附：涉及的yaml文件

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "1"
  creationTimestamp: "2022-06-16T21:09:32Z"
  generation: 1
  labels:
    app: djangoblog
  name: djangoblog
  namespace: default
  resourceVersion: "347847"
  selfLink: /apis/extensions/v1beta1/namespaces/default/deployments/djangoblog
  uid: bcdbd9ef-2bc5-4828-9f21-fae8e80ca503
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: djangoblog
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: djangoblog
    spec:
      containers:
      - env:
        - name: DJANGO_MYSQL_HOST
          value: "10.102.125.157"
        - name: DJANGO_MYSQL_PASSWORD
          value: "123456"
        - name: DJANGO_MYSQL_USER
          value: root
        - name: DJANGO_MYSQL_DATABASE
          value: djangoblog
        image: c104.cheng.com/cheng/djangoblog:v1.1.1
        imagePullPolicy: IfNotPresent
        name: djangoblog
        ports:
        - containerPort: 8000
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
status:
  conditions:
  - lastTransitionTime: "2022-06-16T21:09:32Z"
    lastUpdateTime: "2022-06-16T21:10:12Z"
    message: ReplicaSet "djangoblog-76bb6b66f4" has successfully progressed.
    reason: NewReplicaSetAvailable
    status: "True"
    type: Progressing
  - lastTransitionTime: "2022-06-16T21:10:35Z"
    lastUpdateTime: "2022-06-16T21:10:35Z"
    message: Deployment does not have minimum availability.
    reason: MinimumReplicasUnavailable
    status: "False"
    type: Available
  observedGeneration: 1
  replicas: 1
  unavailableReplicas: 1
  updatedReplicas: 1

安装keepalive

实现 用户访问高可用，安装过程比较简单：略。

主节点配置文件

! Configuration File for keepalived

global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL01
}

vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.18.10/24
}
}

---

从节点配置文件
! Configuration File for keepalived

global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL02
}

vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.18.10/24
}
}

拓展：
安装nginx做端口转发，用户访问80端口即可；增加ssl访问，加密数据。
安装证书：

 1014  openssl genrsa -des3 -out ca.key 2048
 1015  openssl req -new -key ca.key -out server.csr
 1017  mv ca.key ca.key.org
 1018  openssl rsa -in ca.key.org -out ca.key 
 1019  openssl x509 -req -days 36500 -in server.csr -signkey ca.key -out server.crt
 1020  ll
 1021  chmod -R 777 ./*

nginx配置文件：

    server {
        listen       443 ssl http2;
        listen       [::]:443 ssl http2;
        server_name  _;
        root         /usr/share/nginx/html;


            location / {
    proxy_pass  http://10.100.207.173:8000; # 转发规则
    proxy_set_header Host $proxy_host; # 修改转发请求头，让8080端口的应用可以受到真实的请求
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  }





        ssl on;
        ssl_certificate "/root/ssl/server.crt";
        ssl_certificate_key "/root/ssl/ca.key";
        ssl_session_cache shared:SSL:1m;
        ssl_session_timeout  10m;
        ssl_ciphers HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers on;


        include /etc/nginx/default.d/*.conf;

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

登陆web操作页面

通过keepalived的vip访问web节点，查看日志输出三个都有日志输出，并且无页面报错，集群搭建完成。