【kubernetes/k8s源码分析】 k8s ceph rbd storage 源码分析
secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: rbd-secret
type: "kubernetes.io/rbd"
data:
key: QVFBM2IvUmNqQkM2S0JBQVZZL041aFNJSk53algvTDB1eU56Y3c9PQ==
rbd-class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rbd-sc
provisioner: kubernetes.io/rbd
parameters:
monitors: 10.200.112.23:6789
adminId: admin
adminSecretName: rbd-secret
adminSecretNamespace: default
pool: replicapool
userId: admin
userSecretName: rbd-secret
rbd-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rbd-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce
storageClassName: rbd-sc
resources:
requests:
storage: 10G
syncClaim 根据注解 pv.kubernetes.io/bind-completed
--> syncUnboundClaim
--> provisionClaim
--> provisionClaimOperation
--> plugin.NewProvisioner （第 1 章节讲解）
--> provisioner.Provision （第 2 章节讲解）
1. NewProvisioner
实例化 rbdVolumeProvisioner 结构，没啥可说的了
func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
return plugin.newProvisionerInternal(options, &RBDUtil{})
}
func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
return &rbdVolumeProvisioner{
rbdMounter: &rbdMounter{
rbd: newRBD("", "", "", "", false, plugin, manager),
},
options: options,
}, nil
}
2. Provision 函数
验证以及获得 storageclass 参数 parameters，可以参看如上 rbd-class.yaml
if !volutil.AccessModesContainedInAll(r.plugin.GetAccessModes(), r.options.PVC.Spec.AccessModes) {
return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", r.options.PVC.Spec.AccessModes, r.plugin.GetAccessModes())
}
if r.options.PVC.Spec.Selector != nil {
return nil, fmt.Errorf("claim Selector is not supported")
}
var err error
adminSecretName := ""
adminSecretNamespace := rbdDefaultAdminSecretNamespace
secret := ""
secretName := ""
secretNamespace := ""
keyring := ""
imageFormat := rbdImageFormat2
fstype := ""2.1 格式化只能为 1 或者 2，默认为 2
这个格式化，详解看后面ceph pool image format 解析
// sanity check
if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 {
return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s",
imageFormat, rbdImageFormat1, rbdImageFormat2)
}
r.imageFormat = imageFormat2.2 CreateImage 函数
路径 pkg/volume/rbd/rbd_util.go，rbd image是由 kubernetes-dynamic-pv 加上随机UUID生成
rbd create kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15 --size xxx --pool xxxx --id xxx -m xxx --key=xxx -image-format xxxx
func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, size int, err error) {
。。。。。。
args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
if p.rbdMounter.imageFormat == rbdImageFormat2 {
// If no image features is provided, it results in empty string
// which disable all RBD image format 2 features as expected.
features := strings.Join(p.rbdMounter.imageFeatures, ",")
args = append(args, "--image-feature", features)
}
output, err = p.exec.Run("rbd", args...)2.3 pv 注解设置 kubernetes.io/createdby: rbd-dynamic-provisioner
metadata:
annotations:
kubernetes.io/createdby: rbd-dynamic-provisioner
pv := new(v1.PersistentVolume)
metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")
if secretName != "" {
rbd.SecretRef = new(v1.SecretReference)
rbd.SecretRef.Name = secretName
rbd.SecretRef.Namespace = secretNamespace
} else {
var filePathRegex = regexp.MustCompile(`^(?:/[^/!;` + "`" + ` ]+)+$`)
if keyring != "" && !filePathRegex.MatchString(keyring) {
return nil, fmt.Errorf("keyring field must contain a path to a file")
}
rbd.Keyring = keyring
}2.4 设置 pv 信息
具体可以查看如下 pv yaml 文件
rbd.RadosUser = r.Id
rbd.FSType = fstype
pv.Spec.PersistentVolumeSource.RBD = rbd
pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
if len(pv.Spec.AccessModes) == 0 {
pv.Spec.AccessModes = r.plugin.GetAccessModes()
}
pv.Spec.Capacity = v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
}
pv.Spec.MountOptions = r.options.MountOptions
pv.Spec.VolumeMode = volumeMode
注意： rbd 插件只支持 ReadWriteOnce 以及 ReadOnlyMany
func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
return []v1.PersistentVolumeAccessMode{
v1.ReadWriteOnce,
v1.ReadOnlyMany,
}
}
reconcile()
--> rc.operationExecutor.VerifyControllerAttachedVolume
--> GenerateVerifyControllerAttachedVolumeFunc
--> MountVolume
--> GenerateMountVolumeFunc
--> NewAttacher （第 3 章节讲解）
--> volumeAttacher.WaitForAttach （第 4 章节讲解）
--> AttachDisk
--> deviceMountableVolumePlugin.NewDeviceMounter （第 5 章节讲解）
-- > volumeDeviceMounter.MountDevice （第 6 章节讲解）
--> volumePlugin.NewMounter （第 7 章节讲解）
--> volumeMounter.CanMount()
--> volumeMounter.Setup （第 8 章节讲解）
3. NewAttacher
NewAttacher 实例化，实现了 Attach WaitForAttach 方法，manager 有 rbdUtil实现
// NewAttacher implements AttachableVolumePlugin.NewAttacher.
func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
return plugin.newAttacherInternal(&RBDUtil{})
}
func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) {
return &rbdAttacher{
plugin: plugin,
manager: manager,
mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
}, nil
}
4. WaitForAttach 函数
主要实现在 AttachDisk，这个由rbdUtil实现
// WaitForAttach implements Attacher.WaitForAttach. It's called by kublet to
// attach volume onto the node.
// This method is idempotent, callers are responsible for retrying on failure.
func (attacher *rbdAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
klog.V(4).Infof("rbd: waiting for attach volume (name: %s) for pod (name: %s, uid: %s)", spec.Name(), pod.Name, pod.UID)
mounter, err := attacher.plugin.createMounterFromVolumeSpecAndPod(spec, pod)
if err != nil {
klog.Warningf("failed to create mounter: %v", spec)
return "", err
}
realDevicePath, err := attacher.manager.AttachDisk(*mounter)
if err != nil {
return "", err
}
klog.V(3).Infof("rbd: successfully wait for attach volume (spec: %s, pool: %s, image: %s) at %s", spec.Name(), mounter.Pool, mounter.Image, realDevicePath)
return realDevicePath, nil
}4.1 AttachDisk
主要功能是attach 到节点，分片段一块块讲解，这个globalPDPath路径格式 kubernetes.io/rbd mounts ${pool} -image ${image}
示例：/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/replicapool-image-kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca- 0800271c9f15
// AttachDisk attaches the disk on the node.
func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) {
var output []byte
globalPDPath := util.MakeGlobalPDName(*b.rbd)
if pathExists, pathErr := volutil.PathExists(globalPDPath); pathErr != nil {
return "", fmt.Errorf("Error checking if path exists: %v", pathErr)
} else if !pathExists {
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
return "", err
}
}4.1.1 waitForPath 函数，不存在路径则试几次创建，rbd 实现在 getRbdDevFromImageAndPool
// Stat a path, if it doesn't exist, retry maxRetries times.
func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
for i := 0; i < maxRetries; i++ {
if i != 0 {
time.Sleep(time.Second)
}
if useNbdDriver {
if devicePath, found := getNbdDevFromImageAndPool(pool, image); found {
return devicePath, true
}
} else {
if devicePath, found := getRbdDevFromImageAndPool(pool, image); found {
return devicePath, true
}
}
}
return "", false
}4.1.1.1 getRbdDevFromImageAndPool
主要是查看 /sys/bus/rbd/devices/X/name 和 /sys/bus/rbd/devices/X/pool 这俩文件，在看看 /dev/rbd/X 是否存在
// Search /sys/bus for rbd device that matches given pool and image.
func getRbdDevFromImageAndPool(pool string, image string) (string, bool) {
// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
sys_path := "/sys/bus/rbd/devices"
if dirs, err := ioutil.ReadDir(sys_path); err == nil {
for _, f := range dirs {
name := f.Name()
// First match pool, then match name.
poolFile := path.Join(sys_path, name, "pool")
poolBytes, err := ioutil.ReadFile(poolFile)
if strings.TrimSpace(string(poolBytes)) != pool {
klog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes))
continue
}
imgFile := path.Join(sys_path, name, "name")
if strings.TrimSpace(string(imgBytes)) != image {
klog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes))
continue
}
// Found a match, check if device exists.
devicePath := "/dev/rbd" + name
if _, err := os.Lstat(devicePath); err == nil {
return devicePath, true
}
}
}
return "", false
}4.1.1.2 未发现 /dev/rbd/X 文件情况
再看看 accessModes类型，如果只是 ReadWriteMany 就不需要查验是否多个pod使用
if !mapped {
// Currently, we don't acquire advisory lock on image, but for backward
// compatibility, we need to check if the image is being used by nodes running old kubelet.
// osd_client_watch_timeout defaults to 30 seconds, if the watcher stays active longer than 30 seconds,
// rbd image does not get mounted and failure message gets generated.
backoff := wait.Backoff{
Duration: rbdImageWatcherInitDelay,
Factor: rbdImageWatcherFactor,
Steps: rbdImageWatcherSteps,
}
needValidUsed := true
if b.accessModes != nil {
// If accessModes only contains ReadOnlyMany, we don't need check rbd status of being used.
if len(b.accessModes) == 1 && b.accessModes[0] == v1.ReadOnlyMany {
needValidUsed = false
}
}4.1.1.3 如果不让多个pod 使用情况，查看rbd 状态，如果有watchers证明被使用了
# rbd status kubernetes-dynamic-pvc-7ca75ce6-85ca-11e9-958a-0800271c9f15 --pool replicapool
Watchers:
watcher=192.168.74.57:0/1080495508 client.14269 cookie=18446462598732840961
if needValidUsed {
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
used, rbdOutput, err := util.rbdStatus(&b)
if err != nil {
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
}
return !used, nil
})
// Return error if rbd image has not become available for the specified timeout.
if err == wait.ErrWaitTimeout {
return "", fmt.Errorf("rbd image %s/%s is still being used", b.Pool, b.Image)
}
// Return error if any other errors were encountered during waiting for the image to become available.
if err != nil {
return "", err
}
}4.1.1.4 执行 rbd map
rbd map ${pool}/${image}
// Execute command to map a rbd device for mounter.
// rbdCmd is driver dependent and either "rbd" or "rbd-nbd".
func execRbdMap(b rbdMounter, rbdCmd string, mon string) ([]byte, error) {
// Commandline: rbdCmd map imgPath ...
// do not change this format - some tools like rbd-nbd are strict about it.
imgPath := fmt.Sprintf("%s/%s", b.Pool, b.Image)
if b.Secret != "" {
return b.exec.Run(rbdCmd,
"map", imgPath, "--id", b.Id, "-m", mon, "--key="+b.Secret)
} else {
return b.exec.Run(rbdCmd,
"map", imgPath, "--id", b.Id, "-m", mon, "-k", b.Keyring)
}
}
5. NewDeviceMounter
太么懒了，直接使用 Attacher
// NewAttacher implements AttachableVolumePlugin.NewAttacher.
func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
return plugin.newAttacherInternal(&RBDUtil{})
}
// NewDeviceMounter implements DeviceMountableVolumePlugin.NewDeviceMounter
func (plugin *rbdPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
return plugin.NewAttacher()
}
6. MountDevice 函数
首先看看是否deviceMountPath已经挂载了，如果已经挂载则不处理
// MountDevice implements Attacher.MountDevice. It is called by the kubelet to
// mount device at the given mount path.
// This method is idempotent, callers are responsible for retrying on failure.
func (attacher *rbdAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
klog.V(4).Infof("rbd: mouting device %s to %s", devicePath, deviceMountPath)
notMnt, err := attacher.mounter.IsLikelyNotMountPoint(deviceMountPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
return err
}
notMnt = true
} else {
return err
}
}
if !notMnt {
return nil
}6.1 获得挂载参数，调用FormatAndMount格式化并进行 mount操作
其实操作命令就是 mount 以及 mkfs.xxx /dev/rbdX
fstype, err := getVolumeSourceFSType(spec)
if err != nil {
return err
}
ro, err := getVolumeSourceReadOnly(spec)
if err != nil {
return err
}
options := []string{}
if ro {
options = append(options, "ro")
}
mountOptions := volutil.MountOptionFromSpec(spec, options...)
err = attacher.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)
7. NewMounter 函数
主要是获得secret，主要函数在newMounterInternal
func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
if err != nil {
return nil, err
}
secret := ""
if len(secretName) > 0 && len(secretNs) > 0 {
// if secret is provideded, retrieve it
kubeClient := plugin.host.GetKubeClient()
if kubeClient == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
if err != nil {
err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
return nil, err
}
for _, data := range secrets.Data {
secret = string(data)
}
}
// Inject real implementations here, test through the internal function.
return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, secret)
}7.1 newMounterInternal
实例化 rbdMounter，包括rbd 信息
func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string) (volume.Mounter, error) {
mon, err := getVolumeSourceMonitors(spec)
return &rbdMounter{
rbd: newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
Mon: mon,
Id: id,
Keyring: keyring,
Secret: secret,
fsType: fstype,
mountOptions: volutil.MountOptionFromSpec(spec),
accessModes: ams,
}, nil
}
8. SetUp 函数
主要在 diskSetUp这个主要函数中
func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
// diskSetUp checks mountpoints and prevent repeated calls
klog.V(4).Infof("rbd: attempting to setup at %s", dir)
err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
if err != nil {
klog.Errorf("rbd: failed to setup at %s %v", dir, err)
}
klog.V(3).Infof("rbd: successfully setup at %s", dir)
return err
}8.1 diskSetUp 函数
查看目录是否已经被挂载了，路径根据 image，
kubernetes.io/rbd mounts ${pool} -image ${image}
// utility to mount a disk based filesystem
func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *int64) error {
globalPDPath := manager.MakeGlobalPDName(*b.rbd)
notMnt, err := mounter.IsLikelyNotMountPoint(globalPDPath)
if err != nil && !os.IsNotExist(err) {
klog.Errorf("cannot validate mountpoint: %s", globalPDPath)
return err
}
if notMnt {
return fmt.Errorf("no device is mounted at %s", globalPDPath)
}8.1.1 主要是获得路径，如下格式
/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/replicapool-image-kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca- 0800271c9f15
// Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/pool-image-image.
func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
// Backward compatibility for the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/pool-image-image.
deprecatedDir := path.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
info, err := os.Stat(deprecatedDir)
if err == nil && info.IsDir() {
// The device mount path has already been created with the deprecated format, return it.
klog.V(5).Infof("Deprecated format path %s found", deprecatedDir)
return deprecatedDir
}
// Return the canonical format path.
return path.Join(host.GetPluginDir(rbdPluginName), mount.MountsInGlobalPDPath, pool+"-image-"+image)
}8.1.2 如果 volPath 没有被挂载，则创建该目录，进行mount 操作
notMnt, err = mounter.IsLikelyNotMountPoint(volPath)
if err != nil && !os.IsNotExist(err) {
klog.Errorf("cannot validate mountpoint: %s", volPath)
return err
}
if !notMnt {
return nil
}
if err := os.MkdirAll(volPath, 0750); err != nil {
klog.Errorf("failed to mkdir:%s", volPath)
return err
}
// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
options := []string{"bind"}
if (&b).GetAttributes().ReadOnly {
options = append(options, "ro")
}
mountOptions := util.JoinMountOptions(b.mountOptions, options)
err = mounter.Mount(globalPDPath, volPath, "", mountOptions)
if err != nil {
klog.Errorf("failed to bind mount:%s", globalPDPath)
return err
}8.1.3 如果不是只读模式，则设置所有权
if !b.ReadOnly {
volume.SetVolumeOwnership(&b, fsGroup)
}
总结：
Provision 这个主要是 rbd create 操作
Attach 主要是 rbd map
MountDevice， mount /dev/rbdX /var/lib/kubelet/plugins/XXXX
SetUp 这个是 mount 操作
pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
kubernetes.io/createdby: rbd-dynamic-provisioner
pv.kubernetes.io/bound-by-controller: "yes"
pv.kubernetes.io/provisioned-by: kubernetes.io/rbd
creationTimestamp: "2019-06-03T01:16:31Z"
finalizers:
- kubernetes.io/pv-protection
name: pvc-306f0fd8-859d-11e9-b873-0800271c9f15
resourceVersion: "316160"
selfLink: /api/v1/persistentvolumes/pvc-306f0fd8-859d-11e9-b873-0800271c9f15
uid: 3889cdb2-859d-11e9-b873-0800271c9f15
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 9537Mi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: rbd-pvc
namespace: default
resourceVersion: "316146"
uid: 306f0fd8-859d-11e9-b873-0800271c9f15
persistentVolumeReclaimPolicy: Delete
rbd:
image: kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15
keyring: /etc/ceph/keyring
monitors:
- 10.200.112.23:6789
pool: replicapool
secretRef:
name: rbd-secret
user: admin
storageClassName: rbd-sc
volumeMode: Filesystem
status:
phase: Bound
ceph的rbd 理解
Ceph支持的特性，以COW（写时复制）的方式从RBD快照创建克隆，在Ceph中被称为快照分层。分层特性允许用户创建多个CEPH RBD克隆实例。这些特性应用于OpenStack等云平台中，使用快照形式保护ceph RBD 镜像，快照是只读的，但COW克隆是完全可以写 ，可以多次来孵化实例，对云平台来说是非常有用的。
Ceph RBD镜像有format-1 和 format-2两种类型，RBD支持这两种类型，但是分层特性COW克隆特性只支持format-2镜像，默认RBD创建的镜像是format-1。（这个在克隆的时候特别重要）
- Linux内核支不支持rbd块设备： modprobe rbd
- rbd create (pool_name)/（rbd_name） --size xxxxxMB就可以创建一个块设备
- 映射块设备到你的机器 rbd map (pool_name)/（rbd_name）
ceph pool image format 解析
一个rbd image实际上包含了多个对象（默认情况下是image_size/4M）
# rbd info replicapool/kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15
rbd image 'kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15':
size 9.3 GiB in 2385 objects
order 22 (4 MiB objects)
snapshot_count: 0
id: 10dd6b8b4567
block_name_prefix: rbd_data.10dd6b8b4567
format: 2
features:
op_features:
flags:
create_timestamp: Mon Jun 3 01:16:31 2019
如果image-format = 2，如上格式 rbd_data.xxxxxx，如果image-format = 1，格式为 rb.0.xxxxxxx
如下一个image对象是由多个object组成的，他们每一个的存储位置可能都不一样
# rados -p replicapool ls | grep rbd_data.10dd6b8b4567
rbd_data.10dd6b8b4567.0000000000000427
rbd_data.10dd6b8b4567.000000000000043b
rbd_data.10dd6b8b4567.000000000000042f
rbd_data.10dd6b8b4567.000000000000042a
rbd_data.10dd6b8b4567.0000000000000420
rbd_data.10dd6b8b4567.0000000000000438
rbd_data.10dd6b8b4567.0000000000000432
rbd_data.10dd6b8b4567.0000000000000320
rbd_data.10dd6b8b4567.000000000000043c
rbd_data.10dd6b8b4567.0000000000000433
rbd_data.10dd6b8b4567.0000000000000620
rbd_data.10dd6b8b4567.0000000000000435
rbd_data.10dd6b8b4567.00000000000000e0
rbd_data.10dd6b8b4567.000000000000042b
rbd_data.10dd6b8b4567.0000000000000422
rbd_data.10dd6b8b4567.000000000000042d
rbd_data.10dd6b8b4567.0000000000000434
rbd_data.10dd6b8b4567.0000000000000421
rbd_data.10dd6b8b4567.0000000000000437
rbd_data.10dd6b8b4567.000000000000043d
rbd_data.10dd6b8b4567.0000000000000436
rbd_data.10dd6b8b4567.0000000000000430
rbd_data.10dd6b8b4567.000000000000042c
rbd_data.10dd6b8b4567.0000000000000429
rbd_data.10dd6b8b4567.0000000000000428
rbd_data.10dd6b8b4567.0000000000000950
rbd_data.10dd6b8b4567.0000000000000000
rbd_data.10dd6b8b4567.0000000000000360
rbd_data.10dd6b8b4567.0000000000000060
rbd_data.10dd6b8b4567.0000000000000400
rbd_data.10dd6b8b4567.00000000000000a0
rbd_data.10dd6b8b4567.0000000000000600
rbd_data.10dd6b8b4567.0000000000000120
rbd_data.10dd6b8b4567.0000000000000800
rbd_data.10dd6b8b4567.0000000000000020
rbd_data.10dd6b8b4567.0000000000000439
rbd_data.10dd6b8b4567.0000000000000425
rbd_data.10dd6b8b4567.0000000000000424
rbd_data.10dd6b8b4567.0000000000000008
rbd_data.10dd6b8b4567.000000000000043a
rbd_data.10dd6b8b4567.0000000000000426
rbd_data.10dd6b8b4567.000000000000042e
rbd_data.10dd6b8b4567.000000000000043e
rbd_data.10dd6b8b4567.0000000000000423
rbd_data.10dd6b8b4567.0000000000000001
rbd_data.10dd6b8b4567.0000000000000200
rbd_data.10dd6b8b4567.0000000000000431
rbd_data.10dd6b8b4567.000000000000043f
# ceph osd map replicapool rbd_data.10dd6b8b4567.000000000000043f
osdmap e19 pool 'replicapool' (1) object 'rbd_data.10dd6b8b4567.000000000000043f' -> pg 1.309ade3f (1.3f) -> up ([1,0], p1) acting ([1,0], p1)
这代表在pool rbd中的 rbd_data.10dd6b8b4567.000000000000043f 这个对象位于 1.3f 这个pg中，并且位于osd1和osd0上（两个副本）
如下为 osd 中 pg 里面的数据内容
[root@node1 1.3f_head]# ll
总用量 4100
-rw-r--r-- 1 root root 0 6月 3 08:53 __head_0000003F__1
-rw-r--r-- 1 root root 4194304 6月 3 09:39 rbd\udata.10dd6b8b4567.000000000000043f__head_309ADE3F__1
[root@node1 1.3f_head]# pwd
/var/lib/rook/osd0/current/1.3f_head