【kubernetes/k8s源码分析】 external nfs storage 源码分析
https://github.com/kubernetes-incubator/external-storage/tree/master/nfs-client
storageclass.yaml
apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: managed-nfs-storage provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME' parameters: archiveOnDelete: "false"
1. Main函数
从pod中获取环境变量，包括 NFS_SERVER，NFS_PATH，PROVISIONER_NAME
func main() {
flag.Parse()
flag.Set("logtostderr", "true")
server := os.Getenv("NFS_SERVER")
if server == "" {
glog.Fatal("NFS_SERVER not set")
}
path := os.Getenv("NFS_PATH")
if path == "" {
glog.Fatal("NFS_PATH not set")
}
provisionerName := os.Getenv(provisionerNameKey)
if provisionerName == "" {
glog.Fatalf("environment variable %s is not set! Please set it.", provisionerNameKey)
}
1.2 使用 incluster 创建k8s client
主要操作的资源为 PV 和 PVC
// Create an InClusterConfig and use it to create a client for the controller
// to use to communicate with Kubernetes
config, err := rest.InClusterConfig()
if err != nil {
glog.Fatalf("Failed to create config: %v", err)
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
glog.Fatalf("Failed to create client: %v", err)
}
// The controller needs to know what the server version is because out-of-tree
// provisioners aren't officially supported until 1.5
serverVersion, err := clientset.Discovery().ServerVersion()
if err != nil {
glog.Fatalf("Error getting server version: %v", err)
}
2. NewProvisionController函数
实例化 ProvisionController 结构，主要是为 PVC 提供 PV 操作
nfsProvisioner 实现了 Provision 和 Delete 方法
// NewProvisionController creates a new provision controller using
// the given configuration parameters and with private (non-shared) informers.
func NewProvisionController(
client kubernetes.Interface,
provisionerName string,
provisioner Provisioner,
kubeVersion string,
options ...func(*ProvisionController) error,
) *ProvisionController {
id, err := os.Hostname()
if err != nil {
glog.Fatalf("Error getting hostname: %v", err)
}claims 与 volumes 队列，也就是 PVC 与 PV 队列，该 controller 主要就是做这个工作的
ratelimiter := workqueue.NewMaxOfRateLimiter(
workqueue.NewItemExponentialFailureRateLimiter(15*time.Second, 1000*time.Second),
&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
)
if !controller.exponentialBackOffOnError {
ratelimiter = workqueue.NewMaxOfRateLimiter(
workqueue.NewItemExponentialFailureRateLimiter(15*time.Second, 15*time.Second),
&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
)
}
controller.claimQueue = workqueue.NewNamedRateLimitingQueue(ratelimiter, "claims")
controller.volumeQueue = workqueue.NewNamedRateLimitingQueue(ratelimiter, "volumes")使用 informer 机制，这个主要是 PVC，Add Update Delete 回调函数
// ----------------------
// PersistentVolumeClaims
claimHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) { controller.enqueueWork(controller.claimQueue, obj) },
UpdateFunc: func(oldObj, newObj interface{}) { controller.enqueueWork(controller.claimQueue, newObj) },
DeleteFunc: func(obj interface{}) { controller.forgetWork(controller.claimQueue, obj) },
}
if controller.claimInformer != nil {
controller.claimInformer.AddEventHandlerWithResyncPeriod(claimHandler, controller.resyncPeriod)
} else {
controller.claimInformer = informer.Core().V1().PersistentVolumeClaims().Informer()
controller.claimInformer.AddEventHandler(claimHandler)
}
controller.claims = controller.claimInformer.GetStore()还有对 PV Storageclass 使用 informer 机制，流程一样一样的
3. Provision 函数
pv 名字为 ${namespace}-${pvcName}-${options.PVName}
pvcNamespace := options.PVC.Namespace
pvcName := options.PVC.Name
pvName := strings.Join([]string{pvcNamespace, pvcName, options.PVName}, "-")在 /persistentvolumes 目录下创建目录，修改权限，这个就是 PV 的NFS 路径
fullPath := filepath.Join(mountPath, pvName)
glog.V(4).Infof("creating path %s", fullPath)
if err := os.MkdirAll(fullPath, 0777); err != nil {
return nil, errors.New("unable to create directory to provision new pv: " + err.Error())
}
os.Chmod(fullPath, 0777)返回 PV，path 就是原来的加上刚创建的目录，这个就是给pod 挂载的NFS路径，这就体现了动态创建目录的本质
path := filepath.Join(p.path, pvName)
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: options.PVName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
AccessModes: options.PVC.Spec.AccessModes,
MountOptions: options.MountOptions,
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
},
PersistentVolumeSource: v1.PersistentVolumeSource{
NFS: &v1.NFSVolumeSource{
Server: p.server,
Path: path,
ReadOnly: false,
},
},
},
}
4. Delete 函数
取得 path，查看在容器中路径是否存在，不存在就没有该路径，报个警，不管了
path := volume.Spec.PersistentVolumeSource.NFS.Path
pvName := filepath.Base(path)
oldPath := filepath.Join(mountPath, pvName)
if _, err := os.Stat(oldPath); os.IsNotExist(err) {
glog.Warningf("path %s does not exist, deletion skipped", oldPath)
return nil
}
// Get the storage class for this volume.
storageClass, err := p.getClassForVolume(volume)如果 archiveOnDelete 为 false，则简单暴力直接删除该目录
// Determine if the "archiveOnDelete" parameter exists.
// If it exists and has a false value, delete the directory.
// Otherwise, archive it.
archiveOnDelete, exists := storageClass.Parameters["archiveOnDelete"]
if exists {
archiveBool, err := strconv.ParseBool(archiveOnDelete)
if err != nil {
return err
}
if !archiveBool {
return os.RemoveAll(oldPath)
}
}如果 archiveOnDelete 为 true，则该给名，前缀为 archived
archivePath := filepath.Join(mountPath, "archived-"+pvName)
glog.V(4).Infof("archiving path %s to %s", oldPath, archivePath)
return os.Rename(oldPath, archivePath)
总结：
可以提供动态存储功能，实现很简单，就是在容器里挂载nfs 至 /persistentvolumes，在这里创建目录，让 pod 挂载 nfs-path / 刚创建的目录，每一个 PVC 创建的 PV 指定一个目录
缺点觉得就是只能挂载一个 NFS，不能动态传入 NFS 的 server 和 path，这样就可以万能接入