k8s1.18 StorageClass 使用rbd-provisioner提供ceph rbd持久化存储

阳光穿透心脏的1/2处 2022-12-01 11:55 335阅读 0赞

rbd-provisioner为kubernetes 1.5+版本提供了类似于`kubernetes.io/rbd`的ceph rbd持久化存储动态配置实现。

一些用户会使用kubeadm来部署集群，或者将kube-controller-manager以容器的方式运行。这种方式下，kubernetes在创建使用ceph rbd pv/pvc时没任何问题，但使用dynamic provisioning自动管理存储生命周期时会报错。提示`"rbd: create volume failed, err: failed to create rbd image: executable file not found in $PATH:"`。

问题来自gcr.io提供的kube-controller-manager容器镜像未打包ceph-common组件，缺少了rbd命令，因此无法通过rbd命令为pod创建rbd image，查了github的相关文章，目前kubernetes官方在kubernetes-incubator/external-storage项目通过External Provisioners的方式来解决此类问题。

本文主要针对该问题，通过rbd-provisioner的方式，解决ceph rbd的dynamic provisioning问题。

**部署rbd-provisioner**

首先得在kubernetes集群中安装rbd-provisioner，github仓库链接https://github.com/kubernetes-incubator/external-storage

[root@k8s01 ~]# git clone https://github.com/kubernetes-incubator/external-storage.git
    [root@k8s01 ~]# cd external-storage/ceph/rbd/deploy
    [root@k8s01 deploy]# NAMESPACE=kube-system
    [root@k8s01 deploy]# sed -r -i "s/namespace: [^ ]+/namespace: $NAMESPACE/g" ./rbac/clusterrolebinding.yaml ./rbac/rolebinding.yaml
    [root@k8s01 deploy]# kubectl -n $NAMESPACE apply -f ./rbac

根据自己需要，修改rbd-provisioner的namespace；

部署完成后检查rbd-provisioner deployment，确保已经正常部署

[root@k8s01 ~]# kubectl describe deployments.apps -n kube-system rbd-provisioner
    Name:               rbd-provisioner
    Namespace:          kube-system
    CreationTimestamp:  Sat, 13 Oct 2018 20:08:45 +0800
    Labels:             app=rbd-provisioner
    Annotations:        deployment.kubernetes.io/revision: 1
                        kubectl.kubernetes.io/last-applied-configuration:
                          { "apiVersion":"extensions/v1beta1","kind":"Deployment","metadata":{ "annotations":{ },"name":"rbd-provisioner","namespace":"kube-system"},"s...
    Selector:           app=rbd-provisioner
    Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
    StrategyType:       Recreate
    MinReadySeconds:    0
    Pod Template:
      Labels:           app=rbd-provisioner
      Service Account:  rbd-provisioner
      Containers:
       rbd-provisioner:
        Image:      quay.io/external_storage/rbd-provisioner:latest
        Port:       <none>
        Host Port:  <none>
        Environment:
          PROVISIONER_NAME:  ceph.com/rbd
        Mounts:              <none>
      Volumes:               <none>
    Conditions:
      Type           Status  Reason
      ----           ------  ------
      Available      True    MinimumReplicasAvailable
    OldReplicaSets:  <none>
    NewReplicaSet:   rbd-provisioner-db574c5c (1/1 replicas created)
    Events:          <none>

**创建StorageClass**

ceph配置

[root@k8s01 ~]# ceph osd pool create rbd 128 ##创建池
    [root@k8s01 ~]# ceph osd pool application enable rbd rbd
    
    [root@k8s01 ~]# ceph auth get-or-create client.kubernetes mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=rbd' ##创建用户
    [root@k8s01 ~]# ceph auth get-or-create client.kubernetes | tee /etc/ceph/ceph.client.kubernetes.keyring ##每个k8s节点执行
    
    [root@k8s01 ~]# ceph auth list |grep client.kubernetes -A 3 ## 
    client.kubernetes
            key: AQB2UkZf2RTVGBAADVaRxeWDWEpC0YssEF6o6A==
            caps: [mon] allow r
            caps: [osd] allow class-read object_prefix rbd_children,allow rwx pool=rbd
            
    [root@k8s01 ~]# ceph auth get-key client.admin | base64  ##获取client.kubernetes base64 key 下面会用到 
    QVFBWHFrTmZrN2NZR1JBQXQySGFJS0syTzVKUEVjMG5YVUltbnc9PQ==
    [root@k8s01 ~]# ceph auth get-key client.kubernetes | base64 ##获取client.admin base64 key 下面会用到
    QVFCMlVrWmYyUlRWR0JBQURWYVJ4ZVdEV0VwQzBZRHdFRjZvNkE9PQ==
    [root@k8s02 ~]# cat /etc/ceph/ceph.client.kubernetes.keyring  ##要保证每台k8s节点有这个文件
    [client.kubernetes]
            key = AQB2UkZf2RTVGBAADVaRxeWDWEpC0YDwEF6o6A==

ceph端配置完毕

创建StorageClass。创建SC前，需要创建相关用户的secret

[root@k8s01 ~]# vi secrets.yaml
    apiVersion: v1
    kind: Secret
    metadata:
      name: ceph-admin-secret
      namespace: kube-system
    type: "kubernetes.io/rbd"
    data:
      # ceph auth get-key client.admin | base64
      key: QVFBWHFrTmZrN2NZR1JBQXQySGFJS0syTzVKUEVjMG5YVUltbnc9PQ==
    ---
    apiVersion: v1
    kind: Secret
    metadata:
      name: ceph-secret
      namespace: kube-system
    type: "kubernetes.io/rbd"
    data:
      # ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
      # ceph auth get-key client.kube | base64
      key: QVFCMlVrWmYyUlRWR0JBQURWYVJ4ZVdEV0VwQzBZRHdFRjZvNkE9PQ==
    
    [root@k8s01 ~]#  kubectl create -f secrets.yaml
    
    [root@k8s01 ~]# vi secrets-default.yaml
    apiVersion: v1
    kind: Secret
    metadata:
      name: ceph-secret
    type: "kubernetes.io/rbd"
    data:
      # ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
      # ceph auth get-key client.kube | base64
      key: QVFCMlVrWmYyUlRWR0JBQURWYVJ4ZVdEV0VwQzBZRHdFRjZvNkE9PQ==
    
    [root@k8s01 ~]#  kubectl create -f secrets-default.yaml -n default

创建secret保存client.admin和client.kube用户的key，client.admin和client.kube用户的secret可以放在kube-system namespace，但如果其他namespace需要使用ceph rbd的dynamic provisioning功能的话，要在相应的namespace创建secret来保存client.kube用户key信息

[root@k8s01 ~]# vi ceph-rbd-sc.yaml
    apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: ceph-rbd
      annotations:
         storageclass.beta.kubernetes.io/is-default-class: "true"
    provisioner: ceph.com/rbd
    parameters:
      monitors: 10.100.22.145:6789,10.100.22.146:6789,10.100.22.147:6789
      adminId: admin
      adminSecretName: ceph-admin-secret
      adminSecretNamespace: kube-system
      pool: rbd
      userId: kubernetes
      userSecretName: ceph-secret
      fsType: ext4
      imageFormat: "2"
      imageFeatures: "layering"
    
    [root@k8s01 ~]#  kubectl create -f  ceph-rbd-sc.yaml

其他设置和普通的ceph rbd StorageClass一致，但provisioner需要设置为`ceph.com/rbd`，不是默认的`kubernetes.io/rbd`，这样rbd的请求将由rbd-provisioner来处理

考虑到兼容性，建议尽量关闭rbd image feature，并且kubelet节点的ceph-common版本尽量和ceph服务器端保持一致，我的环境使用的14.2.11版本

[root@k8s01 ceph]# rpm -qa |grep ceph-common
    ceph-common-14.2.11-0.el7.x86_64
    
    [root@k8s01 ceph]# kubectl get storageclasses.storage.k8s.io 
    NAME                 PROVISIONER    RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
    ceph-rbd (default)   ceph.com/rbd   Delete          Immediate           false                  21m

**测试ceph rbd自动分配**

在kube-system和default namespace分别创建pod，通过启动一个busybox实例，将ceph rbd镜像挂载到`/usr/share/busybox`

[root@k8s01 ~]# vi test-pod.yaml
    apiVersion: v1
    kind: Pod
    metadata:
      name: ceph-pod1
    spec:
      containers:
      - name: ceph-busybox
        image: busybox
        command: ["sleep", "60000"]
        volumeMounts:
        - name: ceph-vol1
          mountPath: /usr/share/busybox
          readOnly: false
      volumes:
      - name: ceph-vol1
        persistentVolumeClaim:
          claimName: ceph-claim
    ---
    kind: PersistentVolumeClaim
    apiVersion: v1
    metadata:
      name: ceph-claim
    spec:
      accessModes:  
        - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi
    
    [root@k8s01 ~]# kubectl create -f test-pod.yaml -n kube-system
    pod/ceph-pod1 created
    persistentvolumeclaim/ceph-claim created
    [root@k8s01 ~]# kubectl create -f test-pod.yaml -n default
    pod/ceph-pod1 created
    persistentvolumeclaim/ceph-claim created

检查pv和pvc的创建状态，是否都已经创建

[root@k8s01 ceph]# kubectl get pv,pvc
    NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                    STORAGECLASS   REASON   AGE
    persistentvolume/pvc-b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4   1Gi        RWO            Delete           Bound    default/ceph-claim       ceph-rbd                23m
    persistentvolume/pvc-e1b4bdac-7b55-4126-b0cc-f934b46f0dbd   1Gi        RWO            Delete           Bound    kube-system/ceph-claim   ceph-rbd                20s
    
    NAME                               STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    persistentvolumeclaim/ceph-claim   Bound    pvc-b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4   1Gi        RWO            ceph-rbd       23m

在ceph服务器上，检查rbd镜像创建情况和镜像的信息

[root@k8s01 ceph]# rbd ls --pool rbd
    kubernetes-dynamic-pvc-c3c2d52b-e796-11ea-bb7c-96ab287be34a
    kubernetes-dynamic-pvc-0fb1f715-e79a-11ea-bb7c-96ab287be34a
    
    [root@k8s01 ceph]#  rbd info rbd/kubernetes-dynamic-pvc-c3c2d52b-e796-11ea-bb7c-96ab287be34a
    rbd image 'kubernetes-dynamic-pvc-c3c2d52b-e796-11ea-bb7c-96ab287be34a':
            size 1 GiB in 256 objects
            order 22 (4 MiB objects)
            snapshot_count: 0
            id: 100726b8b4567
            block_name_prefix: rbd_data.100726b8b4567
            format: 2
            features: layering
            op_features: 
            flags: 
            create_timestamp: Wed Aug 26 20:22:10 2020
            access_timestamp: Wed Aug 26 20:22:10 2020
            modify_timestamp: Wed Aug 26 20:22:10 2020
    [root@k8s01 ceph]#  rbd info rbd/kubernetes-dynamic-pvc-0fb1f715-e79a-11ea-bb7c-96ab287be34a
    rbd image 'kubernetes-dynamic-pvc-0fb1f715-e79a-11ea-bb7c-96ab287be34a':
            size 1 GiB in 256 objects
            order 22 (4 MiB objects)
            snapshot_count: 0
            id: da766b8b4567
            block_name_prefix: rbd_data.da766b8b4567
            format: 2
            features: layering
            op_features: 
            flags: 
            create_timestamp: Wed Aug 26 20:45:46 2020
            access_timestamp: Wed Aug 26 20:45:46 2020
            modify_timestamp: Wed Aug 26 20:45:46 2020

检查busybox内的文件系统挂载和使用情况，确认能正常工作

[root@k8s01 ~]# kubectl exec -it ceph-pod1 mount |grep rbd
    /dev/rbd0 on /usr/share/busybox type ext4 (rw,seclabel,relatime,stripe=1024,data=ordered)
    [root@k8s01 ~]# kubectl exec -it -n kube-system ceph-pod1 mount |grep rbd
    /dev/rbd0 on /usr/share/busybox type ext4 (rw,seclabel,relatime,stripe=1024,data=ordered)
    
    [root@k8s01 ~]# kubectl exec -it -n kube-system ceph-pod1 df |grep rbd
    /dev/rbd0              1998672      6144   1976144   0% /usr/share/busybox
    [root@k8s01 ~]# kubectl exec -it ceph-pod1 df |grep rbd
    /dev/rbd0              1998672      6144   1976144   0% /usr/share/busybox

测试删除pod能否自动删除pv和pvc，生产环境中谨慎，设置好回收策略

[root@k8s01 ~]# kubectl delete -f test-pod.yaml
    pod "ceph-pod1" deleted
    persistentvolumeclaim "ceph-claim" deleted
    
    [root@k8s01 ~]# kubectl delete -f test-pod.yaml -n kube-system
    pod "ceph-pod1" deleted
    persistentvolumeclaim "ceph-claim" deleted
    
    [root@k8s01 ~]# kubectl get pv
    No resources found.
    [root@k8s01 ~]# kubectl get pvc
    No resources found.
    [root@k8s01 ~]# kubectl get pvc -n kube-system
    No resources found.

**请注意：删除测试 rbd 设备会被删除**

可以在StorageClass中设置删除策略`reclaimPolicy: Retain`

ceph服务器上的rbd image也已清除，自动回收成功

[root@k8s01 ~]# rbd ls --pool rbd

确认之前创建的rbd images都已经删除

**总结**

大部分情况下，我们无需使用rbd provisioner来提供ceph rbd的dynamic provisioning能力。经测试，在OpenShift、Rancher、SUSE CaaS以及本Handbook的二进制文件方式部署，在安装好ceph-common软件包的情况下，定义StorageClass时使用`kubernetes.io/rbd`即可正常使用ceph rbd provisioning功能

**问题**

1.auth: unable to find a keyring on /etc/ceph/ceph.client.kube.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory

报错中找不到/etc/ceph/ceph.client.kube.keyring 手动复制一个文件就好了

每个k8s节点执行以下命令
    [root@k8s03 ~]# cd /etc/ceph/
    [root@k8s03 ceph]# cp ceph.client.admin.keyring ceph.client.kube.keyring

#### pv 和 ceph rbd 对应关系查看 ####

**注意观察kubectl get pv xxxx -o yaml** spec.rbd.image的值

[root@k8s01 ceph]# kubectl get pv pvc-b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4 -o yaml
    apiVersion: v1
    kind: PersistentVolume
    metadata:
      annotations:
        pv.kubernetes.io/provisioned-by: ceph.com/rbd
        rbdProvisionerIdentity: ceph.com/rbd
      creationTimestamp: "2020-08-26T12:22:10Z"
      finalizers:
      - kubernetes.io/pv-protection
      managedFields:
      - apiVersion: v1
        fieldsType: FieldsV1
        fieldsV1:
          f:status:
            f:phase: { }
        manager: kube-controller-manager
        operation: Update
        time: "2020-08-26T12:22:10Z"
      - apiVersion: v1
        fieldsType: FieldsV1
        fieldsV1:
          f:metadata:
            f:annotations:
              .: { }
              f:pv.kubernetes.io/provisioned-by: { }
              f:rbdProvisionerIdentity: { }
          f:spec:
            f:accessModes: { }
            f:capacity:
              .: { }
              f:storage: { }
            f:claimRef:
              .: { }
              f:apiVersion: { }
              f:kind: { }
              f:name: { }
              f:namespace: { }
              f:resourceVersion: { }
              f:uid: { }
            f:persistentVolumeReclaimPolicy: { }
            f:rbd:
              .: { }
              f:fsType: { }
              f:image: { }
              f:keyring: { }
              f:monitors: { }
              f:pool: { }
              f:secretRef:
                .: { }
                f:name: { }
              f:user: { }
            f:storageClassName: { }
            f:volumeMode: { }
        manager: rbd-provisioner
        operation: Update
        time: "2020-08-26T12:22:10Z"
      name: pvc-b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4
      resourceVersion: "17424624"
      selfLink: /api/v1/persistentvolumes/pvc-b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4
      uid: 56e22d57-f95b-484b-b3aa-e37394785fba
    spec:
      accessModes:
      - ReadWriteOnce
      capacity:
        storage: 1Gi
      claimRef:
        apiVersion: v1
        kind: PersistentVolumeClaim
        name: ceph-claim
        namespace: default
        resourceVersion: "17424619"
        uid: b3fd4b59-eb33-4341-bc0f-d5049b7f8ce4
      persistentVolumeReclaimPolicy: Delete
      rbd:
        fsType: ext4
        image: kubernetes-dynamic-pvc-c3c2d52b-e796-11ea-bb7c-96ab287be34a ##注意观察是否和rdb 的image一致
        keyring: /etc/ceph/keyring
        monitors:
        - 10.100.22.145:6789
        - 10.100.22.146:6789
        - 10.100.22.147:6789
        pool: rbd
        secretRef:
          name: ceph-secret
        user: kubernetes
      storageClassName: ceph-rbd
      volumeMode: Filesystem
    status:
      phase: Bound
      
    [root@k8s01 ceph]# rbd ls --pool rbd
    kubernetes-dynamic-pvc-c3c2d52b-e796-11ea-bb7c-96ab287be34a ##和这项一致
    kubernetes-dynamic-pvc-fb708fd7-e783-11ea-9080-be6293f131e0