K8s部署JeecgBoot微服务-编写资源清单文件

最终效果图

创建NFS存储的动态供给

NFS动态供给插件，配置NFS服务器

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  namespace: jeecg-boot
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      namespace: jeecg-boot
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccount: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: registry.cn-beijing.aliyuncs.com/pylixm/nfs-subdir-external-provisioner:v4.0.0
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: k8s-sigs.io/nfs-subdir-external-provisioner
            - name: NFS_SERVER
              value: 192.168.0.2 #NFS服务器地址
            - name: NFS_PATH
              value: /opt  #NFS目录
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.0.2
            path: /opt

K8s的RBAC配置

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: jeecg-boot
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: jeecg-boot
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: jeecg-boot
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: jeecg-boot
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: jeecg-boot
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

创建存储类，名称为nfs-client（重要）

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-client
provisioner: k8s-sigs.io/nfs-subdir-external-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "false"

NACOS使用外置的MYSQL数据库

创建字典信息

---
apiVersion: v1
data:
  mysql.db.name: nacos
  mysql.password: hsj@124Qk
  mysql.port: '3306'
  mysql.service.name: jeecg-boot-mysql
  mysql.user: root
  nacos.password: M2dAYN0sdk3c
kind: ConfigMap
metadata:
  name: nacos-cm
  namespace: jeecg-boot

MYSQL数据库IP为192.168.0.2

---
apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-mysql
  namespace: jeecg-boot              
  labels:
    name: jeecg-boot-mysql           
spec:
  clusterIP: None
  ports:
    - port: 3306
      name: jeecg-boot-mysql          
      targetPort: 3306
---
apiVersion: v1
kind: Endpoints
metadata:
  name: jeecg-boot-mysql
  namespace: jeecg-boot 
  labels:
    name: jeecg-boot-mysql
subsets:
  - addresses:
      - ip: 192.168.0.2        
    ports:
      - port: 3306

NACOS集群资源清单，暴露30848端口到公网

apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-nacos
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-nacos
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - port: 8848
      name: server
      targetPort: 8848
    - port: 9848
      name: client-rpc
      targetPort: 9848
    - port: 9849
      name: raft-rpc
      targetPort: 9849
    - port: 7848
      name: old-raft-rpc
      targetPort: 7848
  selector:
    app: nacos
---    
kind: Service
apiVersion: v1
metadata:
  labels:
    app: jeecg-boot-nacos
  name: nacos-cluster-manage
  namespace: jeecg-boot
spec:
  ports:
  - name: http
    port: 8848
    protocol: TCP
    targetPort: 8848
    nodePort: 30848
  selector:
    app: nacos
  type: NodePort 
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nacos
  namespace: jeecg-boot
spec:
  serviceName: jeecg-boot-nacos
  replicas: 3
  template:
    metadata:
      labels:
        app: nacos
      annotations:
        pod.alpha.kubernetes.io/initialized: "true"
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - nacos
              topologyKey: "kubernetes.io/hostname"
      containers:
        - name: jeecg-boot-nacos
          image: "nacos/nacos-server:1.4.1"
          resources:
            requests:
              memory: "2Gi"
              cpu: "500m"
          ports:
            - containerPort: 8848
              name: client
            - containerPort: 9848
              name: client-rpc
            - containerPort: 9849
              name: raft-rpc
            - containerPort: 7848
              name: old-raft-rpc
          env:
            - name: NACOS_REPLICAS
              value: "3"
            - name: MYSQL_SERVICE_HOST
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.service.name
            - name: MYSQL_SERVICE_DB_NAME
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.db.name
            - name: MYSQL_SERVICE_PORT
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.port
            - name: MYSQL_SERVICE_USER
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.user
            - name: MYSQL_SERVICE_PASSWORD
              valueFrom:
                configMapKeyRef:
                  name: nacos-cm
                  key: mysql.password
            - name: NACOS_SERVER_PORT
              value: "8848"
            - name: NACOS_APPLICATION_PORT
              value: "8848"
            - name: PREFER_HOST_MODE
              value: "hostname"
            - name: NACOS_SERVERS
              value: "nacos-0.jeecg-boot-nacos.jeecg-boot.svc.cluster.local:8848 nacos-1.jeecg-boot-nacos.jeecg-boot.svc.cluster.local:8848 nacos-2.jeecg-boot-nacos.jeecg-boot.svc.cluster.local:8848" 
  selector:
    matchLabels:
      app: nacos

Redis的哨兵集群部署

apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-redis
  namespace: jeecg-boot
  labels:
    app: redis-sentinel
spec:
  ports:
    - name: redis-port
      port: 6379
      targetPort: 6379
      nodePort: 30637
  type: NodePort
  selector:
    app: redis-sentinel
---
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel
  namespace: jeecg-boot
  labels:
    app: redis-sentinel
spec:
  ports:
    - name: redis-port
      port: 26379
      targetPort: 26379
      nodePort: 30037
  type: NodePort
  selector:
    app: redis-sentinel
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: jeecg-boot
spec:
  selector:
    matchLabels:
      app: redis-sentinel
  serviceName: "jeecg-boot-redis"
  replicas: 3
  template:
    metadata:
      labels:
        app: redis-sentinel
    spec:
      initContainers:
        - name: init-redis
          image: redis:5.0.7
          imagePullPolicy: IfNotPresent
          command:
            - bash
            - "-c"
            - |
              set -ex
              # Generate redis server-id from pod ordinal index.
              [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
              ordinal=${BASH_REMATCH[1]}
              cat /mnt/config-map/redis.conf > /mnt/config/redis/redis.conf
              if [[ $ordinal -ne 0 ]]; then
                MASTER_IP=${REDIS_POD_NAME}-0.${REDIS_SERVICE_NAME}
                echo "slaveof $MASTER_IP $REDIS_MASTER_PORT" >> /mnt/config/redis/redis.conf
              else
                MASTER_IP=${SERVER_IP}
              fi
              cat>>/mnt/config/redis/redis.conf<<EOF
              port 6379
              requirepass $REDIS_PWD
              masterauth $REDIS_PWD
              logfile redis.log
              EOF
              cat>/mnt/config/sentinel/redis.conf<<EOF
              port 26379
              sentinel monitor $REDIS_MASTER_NAME $MASTER_IP $REDIS_MASTER_PORT $QUORUM
              sentinel down-after-milliseconds $REDIS_MASTER_NAME $DOWN_AFTER
              sentinel failover-timeout $REDIS_MASTER_NAME $FAILOVER_TIMEOUT
              sentinel parallel-syncs $REDIS_MASTER_NAME $PARALLEL_SYNCS
              sentinel auth-pass $REDIS_MASTER_NAME $REDIS_PWD
              EOF
          env:
            - name: REDIS_PWD
              valueFrom:
                secretKeyRef:
                  name: redis-secret
                  key: REDIS_PWD
            - name: REDIS_POD_NAME
              value: redis
            - name: REDIS_SERVICE_NAME
              value: jeecg-boot-redis
            - name: SERVER_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          envFrom:
            - configMapRef:
                name: redis-config
          volumeMounts:
            - name: redis-conf
              mountPath: /mnt/config/redis
            - name: sentinel-conf
              mountPath: /mnt/config/sentinel
            - name: config-map
              mountPath: /mnt/config-map
      containers:
        - name: redis
          image: redis:5.0.7
          imagePullPolicy: IfNotPresent
          command:
            - bash
            - "-c"
            - |
              exec redis-server /usr/local/etc/redis/redis.conf
          env:
            - name: REDIS_PWD
              valueFrom:
                secretKeyRef:
                  name: redis-secret
                  key: REDIS_PWD
          envFrom:
            - configMapRef:
                name: redis-config
          ports:
            - name: redis
              containerPort: 6379
          volumeMounts:
            - name: data
              mountPath: /data
              subPath: redis
            - name: redis-conf
              mountPath: /usr/local/etc/redis
          resources:
            requests:
              cpu: 500m
              memory: 512Mi
          livenessProbe:
            exec:
              command:
                - bash
                - "-c"
                - |
                  set -ex
                  redis-cli -p $REDIS_MASTER_PORT -a $REDIS_PWD ping
            initialDelaySeconds: 30
            periodSeconds: 5
            timeoutSeconds: 5
          readinessProbe:
            exec:
              # Check we can execute queries over TCP (skip-networking is off).
              command:
                - bash
                - "-c"
                - |
                  set -ex
                  redis-cli -p $REDIS_MASTER_PORT -a $REDIS_PWD ping
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
        - name: sentinel
          image: redis:5.0.7
          imagePullPolicy: IfNotPresent
          command:
            - bash
            - "-c"
            - |
              set -ex
              until redis-cli -p 6379 -a $REDIS_PWD info replication; do
              echo "Waiting for redis to be ready (accepting connections)"
              sleep 5
              done
              echo "Initializing replication from clone position"
              redis-server /usr/local/etc/redis/redis.conf --sentinel
          ports:
            - name: sentinel
              containerPort: 26379
          env:
            - name: REDIS_PWD
              valueFrom:
                secretKeyRef:
                  name: redis-secret
                  key: REDIS_PWD
          envFrom:
            - configMapRef:
                name: redis-config
          volumeMounts:
            - name: data
              mountPath: /data
              subPath: sentinel
            - name: sentinel-conf
              mountPath: /usr/local/etc/redis/
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
          livenessProbe:
            exec:
              command:
                - bash
                - "-c"
                - |
                  set -ex
                  redis-cli -p 26379 -a $REDIS_PWD ping
            initialDelaySeconds: 30
            periodSeconds: 5
            timeoutSeconds: 5
          readinessProbe:
            exec:
              command:
                - bash
                - "-c"
                - |
                  set -ex
                  redis-cli -p 26379 -a $REDIS_PWD ping
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
      volumes:
        - name: redis-conf
          emptyDir: {}
        - name: sentinel-conf
          emptyDir: {}
        - name: config-map
          configMap:
            name: redis-config
  volumeClaimTemplates:
    - metadata:
        name: data
        namespace: jeecg-boot
      spec:
        accessModes:
        - ReadWriteMany
        storageClassName: "nfs-client"
        resources:
          requests:
            storage: 6Gi

RabbitMQ的集群部署

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: rabbitmq-config
  namespace: jeecg-boot
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
    enabled_plugins: |
      [rabbitmq_management,rabbitmq_peer_discovery_k8s,rabbitmq_delayed_message_exchange].
    rabbitmq.conf: |
      default_user = admin
      default_pass = BCw3kkwHFVNj
      cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
      cluster_formation.k8s.address_type = hostname
      cluster_formation.node_cleanup.interval = 30
      cluster_formation.node_cleanup.only_log_warning = true
      cluster_partition_handling = autoheal
      queue_master_locator=min-masters
      loopback_users.guest = false
      cluster_formation.randomized_startup_delay_range.min = 0
      cluster_formation.randomized_startup_delay_range.max = 2
      vm_memory_high_watermark.absolute = 2GB
      disk_free_limit.absolute = 4GB
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
rules:
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jeecg-boot-rabbitmq
subjects:
- kind: ServiceAccount
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: jeecg-boot-rabbitmq
    type: LoadBalancer
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
spec:
  clusterIP: None
  ports:
  - name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app: jeecg-boot-rabbitmq
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: jeecg-boot-rabbitmq
  name: rabbitmq-cluster-manage
  namespace: jeecg-boot
spec:
  ports:
  - name: http
    port: 15672
    protocol: TCP
    targetPort: 15672
    nodePort: 30672
  selector:
    app: jeecg-boot-rabbitmq
  type: NodePort      
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  labels:
    app: jeecg-boot-rabbitmq
  name: jeecg-boot-rabbitmq
  namespace: jeecg-boot
spec:
  replicas: 3
  selector:
    matchLabels:
      app: jeecg-boot-rabbitmq
  serviceName: jeecg-boot-rabbitmq
  template:
    metadata:
      namespace: jeecg-boot
      labels:
        app: jeecg-boot-rabbitmq
    spec:
      serviceAccountName: jeecg-boot-rabbitmq
      terminationGracePeriodSeconds: 30
      containers:
      - name: jeecg-boot-rabbitmq
        image: rabbitmq:3.8.3-management
        imagePullPolicy: IfNotPresent
        args:
        - -c
        - cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
          rabbitmq-server
        command:
        - sh
        env:
        - name: TZ
          value: 'Asia/Shanghai'
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: RABBITMQ_USE_LONGNAME
          value: "true"
        - name: RABBITMQ_ERLANG_COOKIE
          value: 'SWvCP0Hrqv43NG7GybHC95ntCJKoW8UyNFWnBEWG8TY='
        - name: K8S_SERVICE_NAME
          value: jeecg-boot-rabbitmq
        - name: RABBITMQ_NODENAME
          value: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
        - name: K8S_HOSTNAME_SUFFIX
          value: .$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
        - name: RABBITMQ_CONFIG_FILE
          value: /var/lib/rabbitmq/rabbitmq.conf
        livenessProbe:
          exec:
            command:
            - rabbitmq-diagnostics
            - status
          # See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.
          initialDelaySeconds: 60
          periodSeconds: 60
          timeoutSeconds: 15
        ports:
        - name: http
          containerPort: 15672
          protocol: TCP
        - name: amqp
          containerPort: 5672
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - rabbitmq-diagnostics
            - status
          initialDelaySeconds: 20
          periodSeconds: 60
          timeoutSeconds: 10
        volumeMounts:
        - mountPath: /etc/rabbitmq
          name: config-volume
          readOnly: false
        - mountPath: /var/lib/rabbitmq
          name: rabbitmq-storage
          readOnly: false
        - name: timezone
          mountPath: /etc/localtime
          readOnly: true
      volumes:
      - name: config-volume
        configMap:
          name: rabbitmq-config
          items:
          - key: rabbitmq.conf
            path: rabbitmq.conf
          - key: enabled_plugins
            path: enabled_plugins
      - name: timezone
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
  volumeClaimTemplates:
  - metadata:
      name: rabbitmq-storage
      namespace: jeecg-boot
    spec:
      accessModes:
      - ReadWriteMany
      storageClassName: "nfs-client"
      resources:
        requests:
          storage: 6Gi

GateWay网关的资源清单

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jeecg-boot-gateway
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-gateway
spec:
  replicas: 3
  selector:
    matchLabels:
      app: jeecg-boot-gateway
  template:
    metadata:
      labels:
        app: jeecg-boot-gateway
    spec:
      containers:
      - name: jeecg-boot-gateway
        image: 192.168.0.7:50080/repository/jeecg-boot-gateway:v1.0.0
        imagePullPolicy: Always
        ports:
        - containerPort: 9999
        env:
        - name: NACOS-PASS
          valueFrom:
            configMapKeyRef:
              name: nacos-cm
              key: nacos.password
      imagePullSecrets:
      - name: harbor     
---
apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-gateway
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-gateway
spec:
  selector:
    app: jeecg-boot-gateway
  ports:
  - name: http
    port: 9999
    protocol: TCP
    targetPort: 9999
    nodePort: 30099
  type: NodePort

System模块资源清单文件

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jeecg-boot-system
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-system
spec:
  replicas: 3
  selector:
    matchLabels:
      app: jeecg-boot-system
  template:
    metadata:
      labels:
        app: jeecg-boot-system
    spec:
      containers:
      - name: jeecg-boot-system
        image: 192.168.0.7:50080/repository/jeecg-boot-system:v1.0.0
        imagePullPolicy: Always
        ports:
        - containerPort: 7001
        env:
        - name: NACOS-PASS
          valueFrom:
            configMapKeyRef:
              name: nacos-cm
              key: nacos.password
      imagePullSecrets:
      - name: harbor  
---
apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-system
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-system
spec:
  selector:
    app: jeecg-boot-system
  ports:
  - name: http
    port: 7001
    targetPort: 7001
  type: ClusterIP

其他微服务资源清单示例（数据清洗服务）

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jeecg-boot-ind
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-ind
spec:
  replicas: 3
  selector:
    matchLabels:
      app: jeecg-boot-ind
  template:
    metadata:
      labels:
        app: jeecg-boot-ind
    spec:
      imagePullSecrets:
      - name: harbor
      volumes:
      - name: temp
        hostPath:
          path: /opt/temp
      - name: workspace
        hostPath:
          path: /opt/workspace
      - name: up-files
        hostPath:
          path: /opt/upFiles
      containers:
      - name: jeecg-boot-ind
        image: 192.168.0.7:50080/repository/jeecg-boot-ind:v1.0.0
        imagePullPolicy: Always
        ports:
        - containerPort: 7003
        volumeMounts:
        - name: temp
          mountPath: /opt/temp
        - name: workspace
          mountPath: /opt/workspace
        - name: up-files
          mountPath: /opt/upFiles
        env:
        - name: NACOS-PASS
          valueFrom:
            configMapKeyRef:
              name: nacos-cm
              key: nacos.password
---
apiVersion: v1
kind: Service
metadata:
  name: jeecg-boot-ind
  namespace: jeecg-boot
  labels:
    app: jeecg-boot-ind
spec:
  selector:
    app: jeecg-boot-ind
  ports:
  - name: http
    port: 7003
    targetPort: 7003
  type: ClusterIP