Kubernetes使用nfs做为动态存储

机器环境准备

使用StorageClass为k8s作为动态存储，大规模集群中可能会有很多PV，如果这些PV都需要运维手动来处理这也是一件很繁琐的事情，所以就有了动态供给概念，也就是Dynamic Provisioning。而我们上面的创建的PV都是静态供给方式，也就是Static Provisioning。而动态供给的关键就是StorageClass，它的作用就是创建PV模板，提高工作效率,创PVC时会动态自动创建PV。

服务器	用途
10.4.2.104	nfs服务器
10.4.2.100-105	k8s集群

---

搭建nfs服务器

• 10.4.2.104上搭建nfs服务器

yum install rpcbind
yum -y install nfs-utils
k
cat << EOF >> /etc/hosts
/volumes 10.0.0.0/8(rw,no_root_squash,anonuid=998,anongid=994)
EOF
exportfs -rv
systemctl restart rpcbind
systemctl restart nfs
systemctl enable nfs

• k8s所有节点安装nfs客户端

yum -y install nfs-utils

K8S安装nfs-client

• 创建RABCServiceAccount

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: storage-class
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: storage-class
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
    # replace with namespace where provisioner is deployed
  namespace: storage-class
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: storage-class
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: storage-class
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f rabc.yaml

• 安装nfs-client

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: storage-class
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: guopeiyuan/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes/
          env:
            - name: PROVISIONER_NAME
              value: g.iwgame.com/nfs
            - name: NFS_SERVER
              value: 10.4.2.104
            - name: NFS_PATH
              value: /volumes
      volumes:
        - name: nfs-client-root
          nfs:
            server: 10.4.2.104
            path: /volumes

kubectl apply -f deployment.yaml

创建StorageClass

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: iwgame-nfs-storage
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: g.iwgame.com/nfs # or choose another name, must match deployment's env PROVISIONER_NAME'
reclaimPolicy: Retain
parameters:
  archiveOnDelete: "false"

创建PVC测试

• 编写清单

[root@m1 ~]# cat test-pvc.yaml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: test
  namespace: devops
  labels:
    app: test
spec:
  accessModes: [ "ReadWriteOnce" ]
  storageClassName: iwgame-nfs-storage
  resources:
    requests:
      storage: "10Gi"
[root@m1 ~]#yaml

• 测试正常绑定

root@m1 ~]# kubectl get pvc -n devops
NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS         AGE
gitlab-data   Bound    pvc-a85236af-66c8-4641-b508-8b9fb0adae28   10Gi       RWO            iwgame-nfs-storage   18d
minio         Bound    pvc-f6c1bc58-5999-402d-a5de-2d3f3344e0a7   50Gi       RWO            iwgame-nfs-storage   13d
test          Bound    pvc-f1a33fc2-c42c-4f51-bfec-1a631fd98533   10Gi       RWO            iwgame-nfs-storage   7s

prometheus添加存储

alertmanager-alertmanager.yaml和prometheus-prometheus.yaml配置清单后添加以下内容

  serviceMonitorSelector: {}
  version: v2.17.2
  storage:
    volumeClaimTemplate:
      spec:
        storageClassName: iwgame-nfs-storage
        resources:
          requests:
            storage: 50Gi