K8S使用NFS动态-csi-nfs-driver

曾经终败给现在 2024-03-27 10:10 71阅读 0赞

## ***K8S使用NFS动态(csi-nfs-driver)*** ##

nfs服务器：

[root@localhost ~]#yum install nfs-utils rpcbind -y
    [root@localhost ~]#mkdir -p /data/nfs/
    [root@localhost ~]#chmod 755 /data/nfs/
    [root@localhost ~]#vim /etc/exports
    /data/nfs/    *(insecure,rw,sync,no_root_squash,no_all_squash)    //nfs挂载目录；*：挂载地址(*代表所有)
    [root@localhost ~]#systemctl start nfs
    [root@localhost ~]#systemctl start rpcbind
    
    [root@k8s-node1 ~]#yum install nfs-utils rpcbind -y  //k8snode节点都需安装

注：关闭防火墙和防护

k8s-master部署:

rbac-csi-nfs-controller.yaml

---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: csi-nfs-controller-sa
      namespace: kube-system
     
    ---
     
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: nfs-external-provisioner-role
    rules:
      - apiGroups: [""]
        resources: ["persistentvolumes"]
        verbs: ["get", "list", "watch", "create", "delete"]
      - apiGroups: [""]
        resources: ["persistentvolumeclaims"]
        verbs: ["get", "list", "watch", "update"]
      - apiGroups: ["storage.k8s.io"]
        resources: ["storageclasses"]
        verbs: ["get", "list", "watch"]
      - apiGroups: [""]
        resources: ["events"]
        verbs: ["get", "list", "watch", "create", "update", "patch"]
      - apiGroups: ["storage.k8s.io"]
        resources: ["csinodes"]
        verbs: ["get", "list", "watch"]
      - apiGroups: [""]
        resources: ["nodes"]
        verbs: ["get", "list", "watch"]
      - apiGroups: ["coordination.k8s.io"]
        resources: ["leases"]
        verbs: ["get", "list", "watch", "create", "update", "patch"]
    ---
     
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: nfs-csi-provisioner-binding
    subjects:
      - kind: ServiceAccount
        name: csi-nfs-controller-sa
        namespace: kube-system
    roleRef:
      kind: ClusterRole
      name: nfs-external-provisioner-role
      apiGroup: rbac.authorization.k8s.io

csi-nfs-driverinfo.yaml:

---
    apiVersion: storage.k8s.io/v1beta1
    kind: CSIDriver
    metadata:
      name: nfs.csi.k8s.io
    spec:
      attachRequired: false
      volumeLifecycleModes:
        - Persistent
      podInfoOnMount: true

csi-nfs-node.yaml：

---
    # This YAML file contains driver-registrar & csi driver nodeplugin API objects
    # that are necessary to run CSI nodeplugin for nfs
    kind: DaemonSet
    apiVersion: apps/v1
    metadata:
      name: csi-nfs-node
      namespace: kube-system
    spec:
      selector:
        matchLabels:
          app: csi-nfs-node
      template:
        metadata:
          labels:
            app: csi-nfs-node
        spec:
          hostNetwork: true  # original nfs connection would be broken without hostNetwork setting
          dnsPolicy: ClusterFirstWithHostNet
          containers:
            - name: liveness-probe
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/livenessprobe:v2.1.0
              args:
                - --csi-address=/csi/csi.sock
                - --probe-timeout=3s
                - --health-port=29653
                - --v=5
              volumeMounts:
                - name: socket-dir
                  mountPath: /csi
              resources:
                limits:
                  cpu: 100m
                  memory: 100Mi
                requests:
                  cpu: 10m
                  memory: 20Mi
            - name: node-driver-registrar
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/csi-node-driver-registrar:v2.0.1
              lifecycle:
                preStop:
                  exec:
                    command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
              args:
                - --v=5
                - --csi-address=/csi/csi.sock
                - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
              env:
                - name: KUBE_NODE_NAME
                  valueFrom:
                    fieldRef:
                      fieldPath: spec.nodeName
              volumeMounts:
                - name: socket-dir
                  mountPath: /csi
                - name: registration-dir
                  mountPath: /registration
            - name: nfs
              securityContext:
                privileged: true
                capabilities:
                  add: ["SYS_ADMIN"]
                allowPrivilegeEscalation: true
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/nfsplugin:amd64-linux-canary
              args:
                - "-v=5"
                - "--nodeid=$(NODE_ID)"
                - "--endpoint=$(CSI_ENDPOINT)"
              env:
                - name: NODE_ID
                  valueFrom:
                    fieldRef:
                      fieldPath: spec.nodeName
                - name: CSI_ENDPOINT
                  value: unix:///csi/csi.sock
              imagePullPolicy: "IfNotPresent"
              volumeMounts:
                - name: socket-dir
                  mountPath: /csi
                - name: pods-mount-dir
                  mountPath: /var/lib/kubelet/pods
                  mountPropagation: "Bidirectional"
          volumes:
            - name: socket-dir
              hostPath:
                path: /var/lib/kubelet/plugins/csi-nfsplugin
                type: DirectoryOrCreate
            - name: pods-mount-dir
              hostPath:
                path: /var/lib/kubelet/pods
                type: Directory
            - hostPath:
                path: /var/lib/kubelet/plugins_registry
                type: Directory
              name: registration-dir

csi-nfs-controller.yaml：

---
    kind: Deployment
    apiVersion: apps/v1
    metadata:
      name: csi-nfs-controller
      namespace: kube-system
    spec:
      replicas: 2
      selector:
        matchLabels:
          app: csi-nfs-controller
      template:
        metadata:
          labels:
            app: csi-nfs-controller
        spec:
          serviceAccountName: csi-nfs-controller-sa
          nodeSelector:
            kubernetes.io/os: linux
          priorityClassName: system-cluster-critical
          tolerations:
            - key: "node-role.kubernetes.io/master"
              operator: "Equal"
              value: "true"
              effect: "NoSchedule"
          containers:
            - name: csi-provisioner
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/csi-provisioner:v2.0.4
              args:
                - "-v=5"
                - "--csi-address=$(ADDRESS)"
                - "--leader-election"
              env:
                - name: ADDRESS
                  value: /csi/csi.sock
              volumeMounts:
                - mountPath: /csi
                  name: socket-dir
              resources:
                limits:
                  cpu: 100m
                  memory: 100Mi
                requests:
                  cpu: 10m
                  memory: 20Mi
            - name: liveness-probe
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/livenessprobe:v2.1.0
              args:
                - --csi-address=/csi/csi.sock
                - --probe-timeout=3s
                - --health-port=29652
                - --v=5
              volumeMounts:
                - name: socket-dir
                  mountPath: /csi
              resources:
                limits:
                  cpu: 100m
                  memory: 100Mi
                requests:
                  cpu: 10m
                  memory: 20Mi
            - name: nfs
              image: registry.cn-hangzhou.aliyuncs.com/imagesfromgoogle/nfsplugin:amd64-linux-canary
              securityContext:
                privileged: true
                capabilities:
                  add: ["SYS_ADMIN"]
                allowPrivilegeEscalation: true
              imagePullPolicy: IfNotPresent
              args:
                - "-v=5"
                - "--nodeid=$(NODE_ID)"
                - "--endpoint=$(CSI_ENDPOINT)"
              env:
                - name: NODE_ID
                  valueFrom:
                    fieldRef:
                      fieldPath: spec.nodeName
                - name: CSI_ENDPOINT
                  value: unix:///csi/csi.sock
              volumeMounts:
                - name: pods-mount-dir
                  mountPath: /var/lib/kubelet/pods
                  mountPropagation: "Bidirectional"
                - mountPath: /csi
                  name: socket-dir
              resources:
                limits:
                  cpu: 200m
                  memory: 200Mi
                requests:
                  cpu: 10m
                  memory: 20Mi
          volumes:
            - name: pods-mount-dir
              hostPath:
                path: /var/lib/kubelet/pods
                type: Directory
            - name: socket-dir
              emptyDir: {}

storageclass-nfs.yaml ,**修改以下的nfs的地址和目录**

apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: local-storage
    provisioner: nfs.csi.k8s.io
    parameters:
      server: 20.0.0.21
      share: /data/nfs/
    reclaimPolicy: Delete
    volumeBindingMode: WaitForFirstConsumer
    mountOptions:
      - vers=3
      - noresvport

[root@k8s-master1 ~]#kubectl apply -f csi-nfs-driverinfo.yaml
    [root@k8s-master1 ~]#kubectl apply -f csi-nfs-node.yaml
    [root@k8s-master1 ~]#kubectl apply -f csi-nfs-controller.yaml
    [root@k8s-master1 ~]#kubectl apply -f storageclass-nfs.yaml

测试创建pod，使用local-storage动态创建pvc

apiVersion: apps/v1
    kind: StatefulSet
    metadata:
      name: web
    spec:
      serviceName: "nginx"
      replicas: 2
      selector:
        matchLabels:
          app: nginx
      template:
        metadata:
          labels:
            app: nginx
        spec:
          containers:
          - name: nginx
            image: nginx
            ports:
            - containerPort: 80
              name: web
            volumeMounts:
            - name: www
              mountPath: /usr/share/nginx/html
      volumeClaimTemplates:
      - metadata:
          name: www
        spec:
          accessModes: [ "ReadWriteOnce" ]
          resources:
            requests:
              storage: 5Gi
          storageClassName: local-storage

[root@k8s-master1 ~]#kubectl apply -f storageclass-nfs.yaml
    [root@k8s-master1 ~]#kubectl get pvc
    NAME        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS    AGE
    www-web-0   Bound    pvc-c2ce6bec-0f16-4984-9e07-fb6fa876abbf   5Gi        RWO            local-storage   3s
    [root@k8s-master1 ~]#kubectl get pv
    NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS    REASON   AGE
    pvc-c2ce6bec-0f16-4984-9e07-fb6fa876abbf   5Gi        RWO            Delete           Bound    default/www-web-0   local-storage            4s
    [root@k8s-master1 ~]#kubectl get pod
    NAME    READY   STATUS    RESTARTS   AGE
    web-0   1/1     Running   0          43s
    web-1   1/1     Running   0          25s

nfs服务器查看挂载

[root@localhost ~]#cd /data/nfs/
    [root@localhost nfs]#ll
    总用量 0
    drwxr-xr-x. 2 root root 6 12月  9 16:40 pvc-a6b65499-8acf-4c81-9cae-2df3acbeadec
    drwxr-xr-x. 2 root root 6 12月  9 16:39 pvc-c2ce6bec-0f16-4984-9e07-fb6fa876abbf