【kubernetes/k8s源码分析】 k8s ceph rbd storage 源码分析

短命女 2022-01-23 07:37 228阅读 0赞

###     secret.yaml ###

apiVersion: v1
    kind: Secret
    metadata:
      name: rbd-secret
    type: "kubernetes.io/rbd"
    data:
      key: QVFBM2IvUmNqQkM2S0JBQVZZL041aFNJSk53algvTDB1eU56Y3c9PQ==

###     rbd-class.yaml ###

apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
       name: rbd-sc
    provisioner: kubernetes.io/rbd
    parameters:
      monitors: 10.200.112.23:6789
      adminId: admin
      adminSecretName: rbd-secret
      adminSecretNamespace: default
      pool: replicapool
      userId: admin
      userSecretName: rbd-secret

###     rbd-pvc.yaml ###

kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: rbd-pvc
      namespace: default
    spec:
      accessModes:
         - ReadWriteOnce
      storageClassName: rbd-sc
      resources:
        requests:
          storage: 10G

syncClaim 根据注解 pv.kubernetes.io/bind-completed  
       --> syncUnboundClaim

--> provisionClaim

--> provisionClaimOperation

--> plugin.NewProvisioner （第 1 章节讲解）

--> provisioner.Provision （第 2 章节讲解）

#  1. NewProvisioner #

实例化 rbdVolumeProvisioner 结构，没啥可说的了

func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
    	return plugin.newProvisionerInternal(options, &RBDUtil{})
    }
    
    func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
    	return &rbdVolumeProvisioner{
    		rbdMounter: &rbdMounter{
    			rbd: newRBD("", "", "", "", false, plugin, manager),
    		},
    		options: options,
    	}, nil
    }

# 2. Provision 函数 #

验证以及获得 storageclass 参数 parameters，可以参看如上 rbd-class.yaml

if !volutil.AccessModesContainedInAll(r.plugin.GetAccessModes(), r.options.PVC.Spec.AccessModes) {
    	return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", r.options.PVC.Spec.AccessModes, r.plugin.GetAccessModes())
    }
    
    if r.options.PVC.Spec.Selector != nil {
    	return nil, fmt.Errorf("claim Selector is not supported")
    }
    var err error
    adminSecretName := ""
    adminSecretNamespace := rbdDefaultAdminSecretNamespace
    secret := ""
    secretName := ""
    secretNamespace := ""
    keyring := ""
    imageFormat := rbdImageFormat2
    fstype := ""

###     2.1 格式化只能为 1 或者 2，默认为 2 ###

这个格式化，详解看后面ceph pool image format 解析

// sanity check
    if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 {
    	return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s",
    		imageFormat, rbdImageFormat1, rbdImageFormat2)
    }
    r.imageFormat = imageFormat

###     2.2 CreateImage 函数 ###

路径 pkg/volume/rbd/rbd\_util.go，rbd image是由 kubernetes-dynamic-pv 加上随机UUID生成

rbd create kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15 --size xxx --pool xxxx --id xxx -m xxx --key=xxx -image-format xxxx

func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, size int, err error) {
        。。。。。。
    	args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
    	if p.rbdMounter.imageFormat == rbdImageFormat2 {
    		// If no image features is provided, it results in empty string
    		// which disable all RBD image format 2 features as expected.
    		features := strings.Join(p.rbdMounter.imageFeatures, ",")
    		args = append(args, "--image-feature", features)
    	}
    	output, err = p.exec.Run("rbd", args...)

###     2.3 pv 注解设置 kubernetes.io/createdby: rbd-dynamic-provisioner ###

> metadata:  
>   annotations:  
>     kubernetes.io/createdby: rbd-dynamic-provisioner

pv := new(v1.PersistentVolume)
    metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volutil.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")
    
    if secretName != "" {
    	rbd.SecretRef = new(v1.SecretReference)
    	rbd.SecretRef.Name = secretName
    	rbd.SecretRef.Namespace = secretNamespace
    } else {
    	var filePathRegex = regexp.MustCompile(`^(?:/[^/!;` + "`" + ` ]+)+$`)
    	if keyring != "" && !filePathRegex.MatchString(keyring) {
    		return nil, fmt.Errorf("keyring field must contain a path to a file")
    	}
    	rbd.Keyring = keyring
    }

###     2.4 设置 pv 信息 ###

具体可以查看如下 pv yaml 文件

rbd.RadosUser = r.Id
    rbd.FSType = fstype
    pv.Spec.PersistentVolumeSource.RBD = rbd
    pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
    pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
    if len(pv.Spec.AccessModes) == 0 {
    	pv.Spec.AccessModes = r.plugin.GetAccessModes()
    }
    pv.Spec.Capacity = v1.ResourceList{
    	v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
    }
    pv.Spec.MountOptions = r.options.MountOptions
    pv.Spec.VolumeMode = volumeMode

注意： rbd 插件只支持 ReadWriteOnce 以及 ReadOnlyMany

func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
    	return []v1.PersistentVolumeAccessMode{
    		v1.ReadWriteOnce,
    		v1.ReadOnlyMany,
    	}
    }

reconcile()

-->  rc.operationExecutor.VerifyControllerAttachedVolume

-->  GenerateVerifyControllerAttachedVolumeFunc

--> MountVolume

--> GenerateMountVolumeFunc

--> NewAttacher （第 3 章节讲解）

--> volumeAttacher.WaitForAttach （第 4 章节讲解）

-->  AttachDisk

--> deviceMountableVolumePlugin.NewDeviceMounter （第 5 章节讲解）

-- > volumeDeviceMounter.MountDevice （第 6 章节讲解）

--> volumePlugin.NewMounter  （第 7 章节讲解）

--> volumeMounter.CanMount()

--> volumeMounter.Setup （第 8 章节讲解）

# 3. NewAttacher  #

NewAttacher 实例化，实现了 Attach WaitForAttach 方法，manager 有 rbdUtil实现

// NewAttacher implements AttachableVolumePlugin.NewAttacher.
    func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
    	return plugin.newAttacherInternal(&RBDUtil{})
    }
    func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) {
    	return &rbdAttacher{
    		plugin:  plugin,
    		manager: manager,
    		mounter: volutil.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host),
    	}, nil
    }

# 4. WaitForAttach 函数 #

主要实现在 AttachDisk，这个由rbdUtil实现

// WaitForAttach implements Attacher.WaitForAttach. It's called by kublet to
    // attach volume onto the node.
    // This method is idempotent, callers are responsible for retrying on failure.
    func (attacher *rbdAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
    	klog.V(4).Infof("rbd: waiting for attach volume (name: %s) for pod (name: %s, uid: %s)", spec.Name(), pod.Name, pod.UID)
    	mounter, err := attacher.plugin.createMounterFromVolumeSpecAndPod(spec, pod)
    	if err != nil {
    		klog.Warningf("failed to create mounter: %v", spec)
    		return "", err
    	}
    	realDevicePath, err := attacher.manager.AttachDisk(*mounter)
    	if err != nil {
    		return "", err
    	}
    	klog.V(3).Infof("rbd: successfully wait for attach volume (spec: %s, pool: %s, image: %s) at %s", spec.Name(), mounter.Pool, mounter.Image, realDevicePath)
    	return realDevicePath, nil
    }

###     4.1 AttachDisk ###

主要功能是attach 到节点，分片段一块块讲解，这个globalPDPath路径格式  kubernetes.io/rbd  mounts  $\{pool\}  -image  $\{image\}

示例：/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/replicapool-image-kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-  0800271c9f15

// AttachDisk attaches the disk on the node.
    func (util *RBDUtil) AttachDisk(b rbdMounter) (string, error) {
    	var output []byte
    
    	globalPDPath := util.MakeGlobalPDName(*b.rbd)
    	if pathExists, pathErr := volutil.PathExists(globalPDPath); pathErr != nil {
    		return "", fmt.Errorf("Error checking if path exists: %v", pathErr)
    	} else if !pathExists {
    		if err := os.MkdirAll(globalPDPath, 0750); err != nil {
    			return "", err
    		}
    	}

**    4.1.1 waitForPath 函数，不存在路径则试几次创建，rbd 实现在 getRbdDevFromImageAndPool**

// Stat a path, if it doesn't exist, retry maxRetries times.
    func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
    	for i := 0; i < maxRetries; i++ {
    		if i != 0 {
    			time.Sleep(time.Second)
    		}
    		if useNbdDriver {
    			if devicePath, found := getNbdDevFromImageAndPool(pool, image); found {
    				return devicePath, true
    			}
    		} else {
    			if devicePath, found := getRbdDevFromImageAndPool(pool, image); found {
    				return devicePath, true
    			}
    		}
    	}
    	return "", false
    }

**    4.1.1.1  getRbdDevFromImageAndPool**

**      主要是查看 **/sys/bus/rbd/devices/X/name 和 /sys/bus/rbd/devices/X/pool 这俩文件，在看看 /dev/rbd/X 是否存在

// Search /sys/bus for rbd device that matches given pool and image.
    func getRbdDevFromImageAndPool(pool string, image string) (string, bool) {
    	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
    	sys_path := "/sys/bus/rbd/devices"
    	if dirs, err := ioutil.ReadDir(sys_path); err == nil {
    		for _, f := range dirs {
    			name := f.Name()
    			// First match pool, then match name.
    			poolFile := path.Join(sys_path, name, "pool")
    			poolBytes, err := ioutil.ReadFile(poolFile)
    	
    			if strings.TrimSpace(string(poolBytes)) != pool {
    				klog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes))
    				continue
    			}
    			imgFile := path.Join(sys_path, name, "name")
    	
    			if strings.TrimSpace(string(imgBytes)) != image {
    				klog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes))
    				continue
    			}
    			// Found a match, check if device exists.
    			devicePath := "/dev/rbd" + name
    			if _, err := os.Lstat(devicePath); err == nil {
    				return devicePath, true
    			}
    		}
    	}
    	return "", false
    }

**    4.1.1.2 未发现 /dev/rbd/X 文件情况**

再看看 accessModes类型，如果只是 ReadWriteMany 就不需要查验是否多个pod使用

if !mapped {
    	// Currently, we don't acquire advisory lock on image, but for backward
    	// compatibility, we need to check if the image is being used by nodes running old kubelet.
    	// osd_client_watch_timeout defaults to 30 seconds, if the watcher stays active longer than 30 seconds,
    	// rbd image does not get mounted and failure message gets generated.
    	backoff := wait.Backoff{
    		Duration: rbdImageWatcherInitDelay,
    		Factor:   rbdImageWatcherFactor,
    		Steps:    rbdImageWatcherSteps,
    	}
    	needValidUsed := true
    	if b.accessModes != nil {
    		// If accessModes only contains ReadOnlyMany, we don't need check rbd status of being used.
    		if len(b.accessModes) == 1 && b.accessModes[0] == v1.ReadOnlyMany {
    			needValidUsed = false
    		}
    	}

**    4.1.1.3 如果不让多个pod 使用情况，查看rbd 状态，如果有watchers证明被使用了**

\# rbd status kubernetes-dynamic-pvc-7ca75ce6-85ca-11e9-958a-0800271c9f15 --pool replicapool  
Watchers:  
    watcher=192.168.74.57:0/1080495508 client.14269 cookie=18446462598732840961

if needValidUsed {
    	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
    		used, rbdOutput, err := util.rbdStatus(&b)
    		if err != nil {
    			return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
    		}
    		return !used, nil
    	})
    	// Return error if rbd image has not become available for the specified timeout.
    	if err == wait.ErrWaitTimeout {
    		return "", fmt.Errorf("rbd image %s/%s is still being used", b.Pool, b.Image)
    	}
    	// Return error if any other errors were encountered during waiting for the image to become available.
    	if err != nil {
    		return "", err
    	}
    }

**    4.1.1.4 执行 rbd map**

**      rbd map $\{pool\}/$\{image\}**

// Execute command to map a rbd device for mounter.
    // rbdCmd is driver dependent and either "rbd" or "rbd-nbd".
    func execRbdMap(b rbdMounter, rbdCmd string, mon string) ([]byte, error) {
    	// Commandline: rbdCmd map imgPath ...
    	// do not change this format - some tools like rbd-nbd are strict about it.
    	imgPath := fmt.Sprintf("%s/%s", b.Pool, b.Image)
    	if b.Secret != "" {
    		return b.exec.Run(rbdCmd,
    			"map", imgPath, "--id", b.Id, "-m", mon, "--key="+b.Secret)
    	} else {
    		return b.exec.Run(rbdCmd,
    			"map", imgPath, "--id", b.Id, "-m", mon, "-k", b.Keyring)
    	}
    }

# 5. NewDeviceMounter #

太么懒了，直接使用 Attacher

// NewAttacher implements AttachableVolumePlugin.NewAttacher.
    func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
    	return plugin.newAttacherInternal(&RBDUtil{})
    }
    
    // NewDeviceMounter implements DeviceMountableVolumePlugin.NewDeviceMounter
    func (plugin *rbdPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
    	return plugin.NewAttacher()
    }

# 6. MountDevice 函数 #

首先看看是否deviceMountPath已经挂载了，如果已经挂载则不处理

// MountDevice implements Attacher.MountDevice. It is called by the kubelet to
    // mount device at the given mount path.
    // This method is idempotent, callers are responsible for retrying on failure.
    func (attacher *rbdAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error {
    	klog.V(4).Infof("rbd: mouting device %s to %s", devicePath, deviceMountPath)
    	notMnt, err := attacher.mounter.IsLikelyNotMountPoint(deviceMountPath)
    	if err != nil {
    		if os.IsNotExist(err) {
    			if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
    				return err
    			}
    			notMnt = true
    		} else {
    			return err
    		}
    	}
    	if !notMnt {
    		return nil
    	}

###     6.1 获得挂载参数，调用FormatAndMount格式化并进行 mount操作 ###

其实操作命令就是 mount 以及 mkfs.xxx /dev/rbdX

fstype, err := getVolumeSourceFSType(spec)
    if err != nil {
    	return err
    }
    ro, err := getVolumeSourceReadOnly(spec)
    if err != nil {
    	return err
    }
    options := []string{}
    if ro {
    	options = append(options, "ro")
    }
    mountOptions := volutil.MountOptionFromSpec(spec, options...)
    err = attacher.mounter.FormatAndMount(devicePath, deviceMountPath, fstype, mountOptions)

# 7. NewMounter 函数 #

主要是获得secret，主要函数在newMounterInternal

func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
    	secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace)
    	if err != nil {
    		return nil, err
    	}
    	secret := ""
    	if len(secretName) > 0 && len(secretNs) > 0 {
    		// if secret is provideded, retrieve it
    		kubeClient := plugin.host.GetKubeClient()
    		if kubeClient == nil {
    			return nil, fmt.Errorf("Cannot get kube client")
    		}
    		secrets, err := kubeClient.CoreV1().Secrets(secretNs).Get(secretName, metav1.GetOptions{})
    		if err != nil {
    			err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err)
    			return nil, err
    		}
    		for _, data := range secrets.Data {
    			secret = string(data)
    		}
    	}
    
    	// Inject real implementations here, test through the internal function.
    	return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, secret)
    }

###    7.1 newMounterInternal ###

实例化 rbdMounter，包括rbd 信息

func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, secret string) (volume.Mounter, error) {
    	mon, err := getVolumeSourceMonitors(spec)
    
    	return &rbdMounter{
    		rbd:          newRBD(podUID, spec.Name(), img, pool, ro, plugin, manager),
    		Mon:          mon,
    		Id:           id,
    		Keyring:      keyring,
    		Secret:       secret,
    		fsType:       fstype,
    		mountOptions: volutil.MountOptionFromSpec(spec),
    		accessModes:  ams,
    	}, nil
    }

# 8. SetUp 函数 #

主要在 diskSetUp这个主要函数中

func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
    	// diskSetUp checks mountpoints and prevent repeated calls
    	klog.V(4).Infof("rbd: attempting to setup at %s", dir)
    	err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
    	if err != nil {
    		klog.Errorf("rbd: failed to setup at %s %v", dir, err)
    	}
    	klog.V(3).Infof("rbd: successfully setup at %s", dir)
    	return err
    }

###     8.1 diskSetUp 函数 ###

查看目录是否已经被挂载了，路径根据 image，

kubernetes.io/rbd  mounts  $\{pool\}  -image  $\{image\}

// utility to mount a disk based filesystem
    func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *int64) error {
    	globalPDPath := manager.MakeGlobalPDName(*b.rbd)
    	notMnt, err := mounter.IsLikelyNotMountPoint(globalPDPath)
    	if err != nil && !os.IsNotExist(err) {
    		klog.Errorf("cannot validate mountpoint: %s", globalPDPath)
    		return err
    	}
    	if notMnt {
    		return fmt.Errorf("no device is mounted at %s", globalPDPath)
    	}

8.1.1 主要是获得路径，如下格式

/var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/replicapool-image-kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-  0800271c9f15

// Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/pool-image-image.
    func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
    	// Backward compatibility for the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/pool-image-image.
    	deprecatedDir := path.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
    	info, err := os.Stat(deprecatedDir)
    	if err == nil && info.IsDir() {
    		// The device mount path has already been created with the deprecated format, return it.
    		klog.V(5).Infof("Deprecated format path %s found", deprecatedDir)
    		return deprecatedDir
    	}
    	// Return the canonical format path.
    	return path.Join(host.GetPluginDir(rbdPluginName), mount.MountsInGlobalPDPath, pool+"-image-"+image)
    }

8.1.2 如果 volPath 没有被挂载，则创建该目录，进行mount 操作

notMnt, err = mounter.IsLikelyNotMountPoint(volPath)
    if err != nil && !os.IsNotExist(err) {
    	klog.Errorf("cannot validate mountpoint: %s", volPath)
    	return err
    }
    if !notMnt {
    	return nil
    }
    
    if err := os.MkdirAll(volPath, 0750); err != nil {
    	klog.Errorf("failed to mkdir:%s", volPath)
    	return err
    }
    // Perform a bind mount to the full path to allow duplicate mounts of the same disk.
    options := []string{"bind"}
    if (&b).GetAttributes().ReadOnly {
    	options = append(options, "ro")
    }
    mountOptions := util.JoinMountOptions(b.mountOptions, options)
    err = mounter.Mount(globalPDPath, volPath, "", mountOptions)
    if err != nil {
    	klog.Errorf("failed to bind mount:%s", globalPDPath)
    	return err
    }

8.1.3 如果不是只读模式，则设置所有权

if !b.ReadOnly {
    	volume.SetVolumeOwnership(&b, fsGroup)
    }

## 总结： ##

Provision 这个主要是 rbd create 操作

Attach 主要是 rbd map

MountDevice， mount /dev/rbdX /var/lib/kubelet/plugins/XXXX

SetUp 这个是 mount 操作

###     pvc.yaml ###

> apiVersion: v1  
> kind: PersistentVolume  
> metadata:  
>   annotations:  
>     kubernetes.io/createdby: rbd-dynamic-provisioner  
>     pv.kubernetes.io/bound-by-controller: "yes"  
>     pv.kubernetes.io/provisioned-by: kubernetes.io/rbd  
>   creationTimestamp: "2019-06-03T01:16:31Z"  
>   finalizers:  
>   - kubernetes.io/pv-protection  
>   name: pvc-306f0fd8-859d-11e9-b873-0800271c9f15  
>   resourceVersion: "316160"  
>   selfLink: /api/v1/persistentvolumes/pvc-306f0fd8-859d-11e9-b873-0800271c9f15  
>   uid: 3889cdb2-859d-11e9-b873-0800271c9f15  
> spec:  
>   accessModes:  
>   - ReadWriteOnce  
>   capacity:  
>     storage: 9537Mi  
>   claimRef:  
>     apiVersion: v1  
>     kind: PersistentVolumeClaim  
>     name: rbd-pvc  
>     namespace: default  
>     resourceVersion: "316146"  
>     uid: 306f0fd8-859d-11e9-b873-0800271c9f15  
>   persistentVolumeReclaimPolicy: Delete  
>   rbd:  
>     image: kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15  
>     keyring: /etc/ceph/keyring  
>     monitors:  
>     - 10.200.112.23:6789  
>     pool: replicapool  
>     secretRef:  
>       name: rbd-secret  
>     user: admin  
>   storageClassName: rbd-sc  
>   volumeMode: Filesystem  
> status:  
>   phase: Bound

# ceph的rbd 理解 #

>      Ceph支持的特性，以COW（写时复制）的方式从RBD快照创建克隆，在Ceph中被称为快照分层。分层特性允许用户创建多个CEPH RBD克隆实例。这些特性应用于OpenStack等云平台中，使用快照形式保护ceph RBD 镜像，快照是只读的，但COW克隆是完全可以写 ，可以多次来孵化实例，对云平台来说是非常有用的。
> 
>     Ceph RBD镜像有format-1  和 format-2两种类型，RBD支持这两种类型，但是分层特性COW克隆特性只支持format-2镜像，默认RBD创建的镜像是format-1。（这个在克隆的时候特别重要）

*      Linux内核支不支持rbd块设备： modprobe rbd
 *      rbd create (pool\_name)/（rbd\_name） --size   xxxxxMB就可以创建一个块设备
 *      **映射块设备到你的机器 rbd map** (pool\_name)/（rbd\_name）

## ceph pool image format 解析 ##

一个rbd image实际上包含了多个对象（默认情况下是image\_size/4M）

\# rbd info replicapool/kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15

> rbd image 'kubernetes-dynamic-pvc-371a0915-859d-11e9-94ca-0800271c9f15':  
>     size 9.3 GiB in 2385 objects  
>     order 22 (4 MiB objects)  
>     snapshot\_count: 0  
>     id: 10dd6b8b4567  
> ***    block\_name\_prefix: rbd\_data.10dd6b8b4567***  
>     format: 2  
>     features:   
>     op\_features:   
>     flags:   
>     create\_timestamp: Mon Jun  3 01:16:31 2019

如果image-format = 2，如上格式 **rbd\_data.xxxxxx**，如果image-format = 1，格式为  **rb.0.xxxxxxx**

**   如下**一个image对象是由多个object组成的，他们每一个的存储位置可能都不一样

> **\# rados -p replicapool ls | grep rbd\_data.10dd6b8b4567**  
> rbd\_data.10dd6b8b4567.0000000000000427  
> rbd\_data.10dd6b8b4567.000000000000043b  
> rbd\_data.10dd6b8b4567.000000000000042f  
> rbd\_data.10dd6b8b4567.000000000000042a  
> rbd\_data.10dd6b8b4567.0000000000000420  
> rbd\_data.10dd6b8b4567.0000000000000438  
> rbd\_data.10dd6b8b4567.0000000000000432  
> rbd\_data.10dd6b8b4567.0000000000000320  
> rbd\_data.10dd6b8b4567.000000000000043c  
> rbd\_data.10dd6b8b4567.0000000000000433  
> rbd\_data.10dd6b8b4567.0000000000000620  
> rbd\_data.10dd6b8b4567.0000000000000435  
> rbd\_data.10dd6b8b4567.00000000000000e0  
> rbd\_data.10dd6b8b4567.000000000000042b  
> rbd\_data.10dd6b8b4567.0000000000000422  
> rbd\_data.10dd6b8b4567.000000000000042d  
> rbd\_data.10dd6b8b4567.0000000000000434  
> rbd\_data.10dd6b8b4567.0000000000000421  
> rbd\_data.10dd6b8b4567.0000000000000437  
> rbd\_data.10dd6b8b4567.000000000000043d  
> rbd\_data.10dd6b8b4567.0000000000000436  
> rbd\_data.10dd6b8b4567.0000000000000430  
> rbd\_data.10dd6b8b4567.000000000000042c  
> rbd\_data.10dd6b8b4567.0000000000000429  
> rbd\_data.10dd6b8b4567.0000000000000428  
> rbd\_data.10dd6b8b4567.0000000000000950  
> rbd\_data.10dd6b8b4567.0000000000000000  
> rbd\_data.10dd6b8b4567.0000000000000360  
> rbd\_data.10dd6b8b4567.0000000000000060  
> rbd\_data.10dd6b8b4567.0000000000000400  
> rbd\_data.10dd6b8b4567.00000000000000a0  
> rbd\_data.10dd6b8b4567.0000000000000600  
> rbd\_data.10dd6b8b4567.0000000000000120  
> rbd\_data.10dd6b8b4567.0000000000000800  
> rbd\_data.10dd6b8b4567.0000000000000020  
> rbd\_data.10dd6b8b4567.0000000000000439  
> rbd\_data.10dd6b8b4567.0000000000000425  
> rbd\_data.10dd6b8b4567.0000000000000424  
> rbd\_data.10dd6b8b4567.0000000000000008  
> rbd\_data.10dd6b8b4567.000000000000043a  
> rbd\_data.10dd6b8b4567.0000000000000426  
> rbd\_data.10dd6b8b4567.000000000000042e  
> rbd\_data.10dd6b8b4567.000000000000043e  
> rbd\_data.10dd6b8b4567.0000000000000423  
> rbd\_data.10dd6b8b4567.0000000000000001  
> rbd\_data.10dd6b8b4567.0000000000000200  
> rbd\_data.10dd6b8b4567.0000000000000431  
> rbd\_data.10dd6b8b4567.000000000000043f

\# ceph osd map replicapool rbd\_data.10dd6b8b4567.000000000000043f

> osdmap e19 pool 'replicapool' (1) object 'rbd\_data.10dd6b8b4567.000000000000043f' -> pg 1.309ade3f (1.3f) -> up (\[1,0\], p1) acting (\[1,0\], p1)

这代表在pool rbd中的 **r**bd\_data.10dd6b8b4567.000000000000043f** **这个对象位于 1.3f 这个pg中，并且位于osd1和osd0上（两个副本）

如下为 osd 中 pg 里面的数据内容

> \[root@node1 1.3f\_head\]\# ll  
> 总用量 4100  
> \-rw-r--r-- 1 root root       0 6月   3 08:53 \_\_head\_0000003F\_\_1  
> \-rw-r--r-- 1 root root 4194304 6月   3 09:39 rbd\\udata.10dd6b8b4567.000000000000043f\_\_head\_309ADE3F\_\_1  
> \[root@node1 1.3f\_head\]\# pwd  
> /var/lib/rook/osd0/current/1.3f\_head