【kubernetes/k8s源码分析】 rook operator flexvolume provision 源码分析

    初始化 provisioner，分别为 ceph.rook.io 和 rook.io

    controller.NewProvisionController中provisioner实现为RookVolumeProvisioner'

    定义 rook/pkg/operator/ceph/provisioner/provisioner.go

// Run volume provisioner for each of the supported configurations
for name, vendor := range provisionerConfigs {
	volumeProvisioner := provisioner.New(o.context, vendor)
	pc := controller.NewProvisionController(
		o.context.Clientset,
		name,
		volumeProvisioner,
		serverVersion.GitVersion,
	)
	go pc.Run(stopChan)
	logger.Infof("rook-provisioner %s started using %s flex vendor dir", name, vendor)
}

 

        ProvisionController主要是处理 pv pvc

controller.claimQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "claims")
controller.volumeQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "volumes")

 

    哪里调用Provision 函数呢？ kubernetes中这里

volume, err = provisioner.Provision(selectedNode, allowedTopologies)

 

 

1. Provision 函数

    1.1 createVolume 函数

     创建一个卷，CreateImage调用命令 rbd create replicapool/pvc-c304ca86-70fc-11e9-8a6c-0800271c9f15 --size 30720 --cluster=rook-ceph --conf=/var/lib/rook/rook-ceph/rook-ceph.config --keyring=/var/lib/rook/rook-ceph/client.admin.keyring

// createVolume creates a rook block volume.
func (p *RookVolumeProvisioner) createVolume(image, pool, dataPool string, clusterNamespace string, size int64) (*ceph.CephBlockImage, error) {
	if image == "" || pool == "" || clusterNamespace == "" || size == 0 {
		return nil, fmt.Errorf("image missing required fields (image=%s, pool=%s, clusterNamespace=%s, size=%d)", image, pool, clusterNamespace, size)
	}

	createdImage, err := ceph.CreateImage(p.context, clusterNamespace, image, pool, dataPool, uint64(size))
	if err != nil {
		return nil, fmt.Errorf("Failed to create rook block image %s/%s: %v", pool, image, err)
	}
	logger.Infof("Rook block image created: %s, size = %d", createdImage.Name, createdImage.Size)

	return createdImage, nil
}

    1.2 返回PV信息

pv := &v1.PersistentVolume{
	ObjectMeta: metav1.ObjectMeta{
		Name: imageName,
	},
	Spec: v1.PersistentVolumeSpec{
		PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
		AccessModes:                   options.PVC.Spec.AccessModes,
		Capacity: v1.ResourceList{
			v1.ResourceName(v1.ResourceStorage): quantity,
		},
		PersistentVolumeSource: v1.PersistentVolumeSource{
			FlexVolume: &v1.FlexPersistentVolumeSource{
				Driver: flexdriver,
				FSType: cfg.fstype,
				Options: map[string]string{
					flexvolume.StorageClassKey:     storageClass,
					flexvolume.PoolKey:             cfg.blockPool,
					flexvolume.ImageKey:            imageName,
					flexvolume.ClusterNamespaceKey: cfg.clusterNamespace,
					flexvolume.DataBlockPoolKey:    cfg.dataBlockPool,
				},
			},
		},
	},
}

 

2. Delete 函数

    这要是删除 rbd 操作

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *RookVolumeProvisioner) Delete(volume *v1.PersistentVolume) error {
	logger.Infof("Deleting volume %s", volume.Name)
	if volume.Spec.PersistentVolumeSource.FlexVolume == nil {
		return fmt.Errorf("Failed to delete rook block image %s: %v", volume.Name, "PersistentVolume is not a FlexVolume")
	}
	if volume.Spec.PersistentVolumeSource.FlexVolume.Options == nil {
		return fmt.Errorf("Failed to delete rook block image %s: %v", volume.Name, "PersistentVolume has no image defined for the FlexVolume")
	}
	name := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ImageKey]
	pool := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.PoolKey]
	var clusterns string
	if _, ok := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNamespaceKey]; ok {
		clusterns = volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNamespaceKey]
	} else if _, ok := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNameKey]; ok {
		// Fallback to `clusterName` as it was used in Rook version earlier v0.8
		clusterns = volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNameKey]
	}
	if clusterns == "" {
		return fmt.Errorf("Failed to delete rook block image %s/%s: no clusterNamespace or (deprecated) clusterName option given", pool, volume.Name)
	}
	err := ceph.DeleteImage(p.context, clusterns, name, pool)
	if err != nil {
		return fmt.Errorf("Failed to delete rook block image %s/%s: %v", pool, volume.Name, err)
	}
	logger.Infof("succeeded deleting volume %+v", volume)
	return nil
}

 

   PV 资源如下：

apiVersion: v1
kind: PersistentVolume
metadata:
  annotations:
    pv.kubernetes.io/provisioned-by: ceph.rook.io/block
  creationTimestamp: "2019-05-23T06:50:15Z"
  finalizers:
  - kubernetes.io/pv-protection
  name: pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15
  resourceVersion: "135093"
  selfLink: /api/v1/persistentvolumes/pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15
  uid: 05512a01-7d27-11e9-82d7-0800271c9f15
spec:
  accessModes:
  - ReadWriteOnce
  capacity:
    storage: 20Gi
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: mysql-pv-claim
    namespace: default
    resourceVersion: "135083"
    uid: 03ccc03f-7d27-11e9-82d7-0800271c9f15
  flexVolume:
    driver: ceph.rook.io/rook-ceph
    fsType: xfs
    options:
      clusterNamespace: rook-ceph
      dataBlockPool: ""
      image: pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15
      pool: replicapool
      storageClass: rook-ceph-block
  persistentVolumeReclaimPolicy: Delete
  storageClassName: rook-ceph-block
  volumeMode: Filesystem
status:
  phase: Bound

      

     attach 操作在agent 模块