k8s nfs-csi实践
本文实践k8s csi中的nfs插件安装及使用流程，k8s环境如下
root@master:~# kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master Ready control-plane 250d v1.24.1 192.168.56.2 Ubuntu 21.04 5.11.0-49-generic containerd://1.5.11
node1 Ready 99d v1.24.1 192.168.56.3 Ubuntu 21.04 5.11.0-49-generic containerd://1.5.11
安装nfs-server
在master上安装nfs
apt-get install nfs-kernel-server
设置nfs目录
mkdir -p /srv/nfs
chown nobody:nogroup /srv/nfs
chmod 0777 /srv/nfs
添加管理ip到配置文件/etc/exports中
/srv/nfs 192.168.56.0/24(rw,sync,no_subtree_check)
重启nfs
systemctl restart nfs-kernel-server
node1上挂载nfs文件系统，使node1和master可访问相同的nfs目录
mkdir -p /srv/nfs
mount -t nfs 192.168.56.2:/srv/nfs /srv/nfs
安装csi nfs driver
下载csi-nfs代码，执行安装脚本
git clone https://github.com/kubernetes-csi/csi-driver-nfs.git
cd csi-driver-nfs
root@master:~/csi-driver-nfs# ./deploy/install-driver.sh
Installing NFS CSI driver, version: master ...
serviceaccount/csi-nfs-controller-sa created
serviceaccount/csi-nfs-node-sa created
clusterrole.rbac.authorization.k8s.io/nfs-external-provisioner-role created
clusterrolebinding.rbac.authorization.k8s.io/nfs-csi-provisioner-binding created
csidriver.storage.k8s.io/nfs.csi.k8s.io created
deployment.apps/csi-nfs-controller created
daemonset.apps/csi-nfs-node created
NFS CSI driver installed successfully.
安装成功后，可看到多了deployment和daemonsets
root@master:~/csi-driver-nfs# kubectl get deployment -A
NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
kube-system csi-nfs-controller 1/1 1 1 36s
...
root@master:~/csi-driver-nfs# kubectl get daemonsets -A
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system csi-nfs-node 2 2 2 2 2 kubernetes.io/os=linux 50s
...
root@master:~/csi-driver-nfs# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
...
kube-system csi-nfs-controller-7f547655c8-b8b75 3/3 Running 0 66s
kube-system csi-nfs-node-726p4 3/3 Running 0 64s
kube-system csi-nfs-node-v7dw5 3/3 Running 0 64s
...
使用csi nfs
创建storageclass_test.yaml，provisioner为nfs.csi.k8s.io，parameters为nfs server参数
root@master:~# cat storageclass_test.yaml
allowVolumeExpansion: true
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-nfs-sc
resourceVersion: "862171"
provisioner: nfs.csi.k8s.io
parameters:
server: 192.168.56.2
share: /srv/nfs
reclaimPolicy: Delete
volumeBindingMode: Immediate
创建pvc-test.yaml，指定storageClassName
root@master:~# cat pvc-test.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc-test
namespace: default
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: csi-nfs-sc
创建pod配置文件pod-test.yaml，指定pvc
root@master:~# cat pod-test.yaml
apiVersion: v1
kind: Pod
metadata:
name: my-csi-app1
namespace: default
spec:
containers:
- command:
- sleep
- "1000000"
image: nginx:1.14
imagePullPolicy: IfNotPresent
name: my-frontend
volumeMounts:
- mountPath: /data
name: my-csi-volume
volumes:
- name: my-csi-volume
persistentVolumeClaim:
claimName: pvc-test
应用上面三个yaml文件，创建出pv，pvc和pod
root@master:~# kubectl apply -f storageclass_test.yaml
storageclass.storage.k8s.io/csi-nfs-sc created
root@master:~# kubectl apply -f pvc-test.yaml
persistentvolumeclaim/pvc-test created
root@master:~# kubectl apply -f pod-test.yaml
pod/my-csi-app1 created
root@master:~# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-afaba19e-edf1-40f8-8983-61dfabf180a6 1Gi RWO Delete Bound default/pvc-test csi-nfs-sc 8s
root@master:~# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc-test Bound pvc-afaba19e-edf1-40f8-8983-61dfabf180a6 1Gi RWO csi-nfs-sc 11s
root@master:~# kubectl get pod
NAME READY STATUS RESTARTS AGE
my-csi-app1 1/1 Running 0 10s
登录pod内部，在nfs存储目录创建测试文件 /data/test
root@master:~# kubectl exec -it my-csi-app1 bash
root@my-csi-app1:/# echo hello > /data/test
退出pod，可在node上的nfs目录看到pod中创建的文件，证明csi nfs可正常工作
root@my-csi-app1:/# exit
root@master:~# cat /srv/nfs/pvc-afaba19e-edf1-40f8-8983-61dfabf180a6/test
hello