参考文献
https://kubernetes-csi.github.io/docs/deploying.html
Driver Volume Mounts
The node plugin needs direct access to the host for making block devices and/or filesystem mounts available to the Kubernetes kubelet.
The mount point used by the CSI driver must be set to Bidirectional to allow Kubelet on the host to see mounts created by the CSI driver container. See the example below:
containers:
- name: my-csi-driver
...
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: mountpoint-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: node-driver-registrar
...
volumeMounts:
- name: registration-dir
mountPath: /registration
volumes:
# This volume is where the socket for kubelet->driver communication is done
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/
type: DirectoryOrCreate
# This volume is where the driver mounts volumes
- name: mountpoint-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
# This volume is where the node-driver-registrar registers the plugin
# with kubelet
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
type: Directory
查看集群CSI插件
在集群中查看
root@server29:~# ls /var/lib/kubelet/
device-plugins pki plugins plugins_registry
root@server29:~# ls /var/lib/kubelet/p
pki/ plugins/ plugins_registry/
root@server29:~# ls /var/lib/kubelet/plugins
com.seaweedfs.csi
root@server29:~# ls /var/lib/kubelet/plugins_registry/
seaweedfs-csi-driver-reg.sock
root@server29:~# ls /var/lib/kubelet/pki/
kubelet-client-2019-06-19-15-21-34.pem kubelet-client-current.pem
root@server29:~# ls /var/lib/kubelet/device-plugins/
DEPRECATION kubelet_internal_checkpoint kubelet.sock nvidia.sock rdma-sriov-dp.sock
该节点安装了com.seaweedfs.csi的csi插件
查看
kubelet的启动命令中的根目录--root-dir=/data1/k8s/kubelet
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77
containers etc-hosts plugins volumes
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/containers
zhangbo6-inpainting
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/volumes/
kubernetes.io~empty-dir kubernetes.io~nfs kubernetes.io~secret
root@server29:~# ls /data1/k8s/kubelet/pods/020bb12f-1da6-11eb-85b5-8446fe72fa77/plugins/
kubernetes.io~empty-dir
去查看pod对应的挂载信息发现没有挂载seaweedfs
其中 020bb12f-1da6-11eb-85b5-8446fe72fa77是pod对应的uid
kubectl get pods -o custom-columns=PodName:.metadata.name,PodUID:.metadata.uid |grep 020bb12f-1da6-11eb-85b5-8446fe72fa77
qiantao@qiant ~ % kc get pod zhangbo6-zhangbo6-inpainting -oyaml |grep containerID
- containerID: docker://10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92
在节点上也能看到对应的进程
root@server29:~# ps -aux |grep 10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92
root 36133 0.0 0.0 11788 5108 ? Sl Nov03 0:05 containerd-shim -namespace moby -workdir /data/k8s/k8s_docker/data/containerd/daemon/io.containerd.runtime.v1.linux/moby/10be561c850486cad50d1f7ed12e0e5a93e40d424dd929ab00ecdc048479bf92 -address /var/run/docker/containerd/containerd.sock -containerd-binary /data/services/k8s_docker-GPU-1.18.9.3/bin/containerd -runtime-root /var/run/docker/runtime-nvidia
另外一个例子
当我在pod中启用
apiVersion: v1
kind: Pod
metadata:
labels:
pm-pod-db-id: "9529"
name: qiantao-test-busybox
namespace: default
spec:
containers:
- args:
- sleep
- "100000000"
image: busybox
imagePullPolicy: IfNotPresent
name: test-create4-8
volumeMounts:
- mountPath: /dev/shm
name: dshm
- mountPath: /data/netfs
name: weedfs-volume
nodeSelector:
ip: 10.26.133.29
volumes:
- name: weedfs-volume
persistentVolumeClaim:
claimName: qiantao-pvc-csi-data
- emptyDir:
medium: Memory
name: dshm
PVC的配置如下
qiantao@qiant ~ % kc get pvc qiantao-pvc-csi-data -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-06-18T11:05:21Z"
finalizers:
- kubernetes.io/pvc-protection
name: qiantao-pvc-csi-data
namespace: default
resourceVersion: "614229919"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/qiantao-pvc-csi-data
uid: 9a394cfb-b153-11ea-bcb7-ac1f6b244dd4
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1000Gi
selector:
matchLabels:
pv: qiantao-pv-csi-data1
storageClassName: seaweedfs-storage
volumeMode: Filesystem
volumeName: qiantao-pv-csi-data
status:
accessModes:
- ReadWriteMany
capacity:
storage: 1000Gi
phase: Bound
PV如下所示
qiantao@qiant ~ % kc get pv qiantao-pv-csi-data -o yaml
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
pv.kubernetes.io/provisioned-by: com.seaweedfs.csi
creationTimestamp: "2020-06-18T11:05:21Z"
finalizers:
- kubernetes.io/pv-protection
- external-attacher/seaweedfs-csi-driver
labels:
pv: qiantao-pv-csi-data1
name: qiantao-pv-csi-data
resourceVersion: "614229916"
selfLink: /api/v1/persistentvolumes/qiantao-pv-csi-data
uid: 9a3276ac-b153-11ea-bcb7-ac1f6b244dd4
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 1000Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: qiantao-pvc-csi-data
namespace: default
resourceVersion: "614229912"
uid: 9a394cfb-b153-11ea-bcb7-ac1f6b244dd4
csi:
driver: seaweedfs-csi-driver
fsType: ext4
volumeHandle: csi-qiantao-storage
persistentVolumeReclaimPolicy: Retain
storageClassName: seaweedfs-storage
volumeMode: Filesystem
status:
phase: Bound
最终在pod使用时（pod uid=53fe1c36-1ff9-11eb-8a23-b008758cb2a8）
root@server29:~# tree /data1/k8s/kubelet/pods/53fe1c36-1ff9-11eb-8a23-b008758cb2a8/volumes/ -L 4
/data1/k8s/kubelet/pods/53fe1c36-1ff9-11eb-8a23-b008758cb2a8/volumes/
├── kubernetes.io~csi
│ └── qiantao-pv-csi-data
│ ├── mount
│ │ ├── 111
│ │ ├── 3333
│ │ ├── agedatastes
│ │ ├── beat_game
│ │ ├── data
│ └── vol_data.json
├── kubernetes.io~empty-dir
│ └── dshm
└── kubernetes.io~secret
└── default-token-6lktj
├── ca.crt -> ..data/ca.crt
├── namespace -> ..data/namespace
└── token -> ..data/token
可以看出使用了kubernetes.io~csi类型的存储挂载，当挂载成功后显示了挂载的目录
使用命令看到的目录与文件系统上一致
kc exec -it qiantao-test-busybox -- ls /data/netfs