Centos7 下Kubernetes集群安装部署

我会带着你远行 2022-06-02 04:16 334阅读 0赞

# 操作环境 #

网络拓扑图

![Center][]、

操作环境：

**k8s-master：**

OS：

[root@k8s-master ~]# cat /etc/redhat-release 
    CentOS Linux release 7.4.1708 (Core)

etcd：

[root@k8s-master ~]# etcd --version
    etcd Version: 3.2.9
    Git SHA: f1d7dd8
    Go Version: go1.8.3
    Go OS/Arch: linux/amd64

Kubernetes：

[root@k8s-master ~]# kubectl version
    Client Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"269f928217957e7126dc87e6adfa82242bfe5b1e", GitTreeState:"clean", BuildDate:"2017-07-03T15:31:10Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"}
    Server Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"269f928217957e7126dc87e6adfa82242bfe5b1e", GitTreeState:"clean", BuildDate:"2017-07-03T15:31:10Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"}

**k8s-node1&k8s-node2：**

OS：

[root@k8s-node1 ~]# cat /etc/redhat-release 
    CentOS Linux release 7.4.1708 (Core)

etcd：

[root@k8s-node1 ~]# etcd --version
    etcd Version: 3.2.9
    Git SHA: f1d7dd8
    Go Version: go1.8.3
    Go OS/Arch: linux/amd64

docker：

[root@k8s-node1 ~]# docker version
    Client:
     Version:         1.12.6
     API version:     1.24
     Package version: docker-1.12.6-68.gitec8512b.el7.centos.x86_64
     Go version:      go1.8.3
     Git commit:      ec8512b/1.12.6
     Built:           Mon Dec 11 16:08:42 2017
     OS/Arch:         linux/amd64
    
    Server:
     Version:         1.12.6
     API version:     1.24
     Package version: docker-1.12.6-68.gitec8512b.el7.centos.x86_64
     Go version:      go1.8.3
     Git commit:      ec8512b/1.12.6
     Built:           Mon Dec 11 16:08:42 2017
     OS/Arch:         linux/amd64

Kubernetes：

[root@k8s-node1 ~]# kubectl version
    Client Version: version.Info{Major:"1", Minor:"5", GitVersion:"v1.5.2", GitCommit:"269f928217957e7126dc87e6adfa82242bfe5b1e", GitTreeState:"clean", BuildDate:"2017-07-03T15:31:10Z", GoVersion:"go1.7.4", Compiler:"gc", Platform:"linux/amd64"}

# 安装部署 #

## 安装前准备 ##

在安装部署集群前，先将三台服务器的时间通过NTP进行同步，否则，在后面的运行中可能会提示错误

[root@k8s-node1 ~]# ntpdate -u cn.pool.ntp.org

在node节点上安装redhat-ca.crt

[root@k8s-node1 ~]# yum install *rhsm* -y

## etcd集群配置 ##

### master节点配置 ###

1.安装kubernetes etcd

[root@k8s-master ~]# yum -y install kubernetes-master etcd

2.配置etcd选项

ETCD\_DATA\_DIR /ETCD\_LISTEN\_PEER\_URLS/ETCD\_LISTEN\_CLIENT\_URLS/ETCD\_MAX\_SNAPSHOTS/ETCD\_NAME/ETCD\_INITIAL\_ADVERTISE\_PEERURLS/ETCD\_ADVERTISE\_CLIENT\_URLS/ETCD\_INITIAL\_CLUSTER

[root@k8s-master ~]# vi /etc/etcd/etcd.conf 
    #[Member]
    #ETCD_CORS=""
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    #ETCD_WAL_DIR=""
    ETCD_LISTEN_PEER_URLS="http://10.10.200.224:2380"
    ETCD_LISTEN_CLIENT_URLS="http://10.10.200.224:2379,http://127.0.0.1:2379"
    ETCD_MAX_SNAPSHOTS="5"
    #ETCD_MAX_WALS="5"
    ETCD_NAME="etcd1"
    #ETCD_SNAPSHOT_COUNT="100000"
    #ETCD_HEARTBEAT_INTERVAL="100"
    #ETCD_ELECTION_TIMEOUT="1000"
    #ETCD_QUOTA_BACKEND_BYTES="0"
    #
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.200.224:2380"
    ETCD_ADVERTISE_CLIENT_URLS="http://10.10.200.224:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_DISCOVERY_SRV=""
    ETCD_INITIAL_CLUSTER="etcd1=http://10.10.200.224:2380,etcd2=http://10.10.200.229:2380,etcd3=http://10.10.200.230:2380"
    #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    #ETCD_INITIAL_CLUSTER_STATE="new"
    #ETCD_STRICT_RECONFIG_CHECK="true"
    #ETCD_ENABLE_V2="true"
    #
    #[Proxy]
    #ETCD_PROXY="off"
    #ETCD_PROXY_FAILURE_WAIT="5000"
    #ETCD_PROXY_REFRESH_INTERVAL="30000"
    #ETCD_PROXY_DIAL_TIMEOUT="1000"
    #ETCD_PROXY_WRITE_TIMEOUT="5000"
    #ETCD_PROXY_READ_TIMEOUT="0"
    #
    #[Security]
    #ETCD_CERT_FILE=""
    #ETCD_KEY_FILE=""
    #ETCD_CLIENT_CERT_AUTH="false"
    #ETCD_TRUSTED_CA_FILE=""
    #ETCD_AUTO_TLS="false"
    #ETCD_PEER_CERT_FILE=""
    #ETCD_PEER_KEY_FILE=""
    #ETCD_PEER_CLIENT_CERT_AUTH="false"
    #ETCD_PEER_TRUSTED_CA_FILE=""
    #ETCD_PEER_AUTO_TLS="false"
    #
    #[Logging]
    #ETCD_DEBUG="false"
    #ETCD_LOG_PACKAGE_LEVELS=""
    #ETCD_LOG_OUTPUT="default"
    #
    #[Unsafe]
    #ETCD_FORCE_NEW_CLUSTER="false"
    #
    #[Version]
    #ETCD_VERSION="false"
    #ETCD_AUTO_COMPACTION_RETENTION="0"
    #
    #[Profiling]
    #ETCD_ENABLE_PPROF="false"
    #ETCD_METRICS="basic"
    #
    #[Auth]
    #ETCD_AUTH_TOKEN="simple"

### nodes节点配置 ###

1.安装部署 kubernetes-node/etcd/flannel/docker

[root@k8s-node1 ~]# yum -y install kubernetes-node etcd flannel docker

2.分别配置etcd，node1与node2的配置方法相同，以node1配置文件为例说明

[root@k8s-node1 ~]# vi /etc/etcd/etcd.conf 
    #[Member]
    #ETCD_CORS=""
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    #ETCD_WAL_DIR=""
    ETCD_LISTEN_PEER_URLS="http://10.10.200.229:2380"
    ETCD_LISTEN_CLIENT_URLS="http://10.10.200.229:2379,http://127.0.0.1:2379"
    #ETCD_MAX_SNAPSHOTS="5"
    #ETCD_MAX_WALS="5"
    ETCD_NAME="etcd2"
    #ETCD_SNAPSHOT_COUNT="100000"
    #ETCD_HEARTBEAT_INTERVAL="100"
    #ETCD_ELECTION_TIMEOUT="1000"
    #ETCD_QUOTA_BACKEND_BYTES="0"
    #
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.200.229:2380"
    ETCD_ADVERTISE_CLIENT_URLS="http://10.10.200.229:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_DISCOVERY_SRV=""
    #ETCD_INITIAL_CLUSTER="default=http://localhost:2380"
    #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    "/etc/etcd/etcd.conf" 66L, 1696C
    #[Member]
    #ETCD_CORS=""
    ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
    #ETCD_WAL_DIR=""
    ETCD_LISTEN_PEER_URLS="http://10.10.200.229:2380"
    ETCD_LISTEN_CLIENT_URLS="http://10.10.200.229:2379,http://127.0.0.1:2379"
    #ETCD_MAX_SNAPSHOTS="5"
    #ETCD_MAX_WALS="5"
    ETCD_NAME="etcd2"
    #ETCD_SNAPSHOT_COUNT="100000"
    #ETCD_HEARTBEAT_INTERVAL="100"
    #ETCD_ELECTION_TIMEOUT="1000"
    #ETCD_QUOTA_BACKEND_BYTES="0"
    #
    #[Clustering]
    ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.200.229:2380"
    ETCD_ADVERTISE_CLIENT_URLS="http://10.10.200.229:2379"
    #ETCD_DISCOVERY=""
    #ETCD_DISCOVERY_FALLBACK="proxy"
    #ETCD_DISCOVERY_PROXY=""
    #ETCD_DISCOVERY_SRV=""
    #ETCD_INITIAL_CLUSTER="default=http://localhost:2380"
    #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
    #ETCD_INITIAL_CLUSTER_STATE="new"
    #ETCD_STRICT_RECONFIG_CHECK="true"
    #ETCD_ENABLE_V2="true"
    ETCD_INITIAL_CLUSTER="etcd1=http://10.10.200.224:2380,etcd2=http://10.10.200.229:2380,etcd3=http://10.10.200.230:2380"
    #
    #[Proxy]
    #ETCD_PROXY="off"
    #ETCD_PROXY_FAILURE_WAIT="5000"
    #ETCD_PROXY_REFRESH_INTERVAL="30000"
    #ETCD_PROXY_DIAL_TIMEOUT="1000"
    #ETCD_PROXY_WRITE_TIMEOUT="5000"
    #ETCD_PROXY_READ_TIMEOUT="0"
    #
    #[Security]
    #ETCD_CERT_FILE=""
    #ETCD_KEY_FILE=""
    #ETCD_CLIENT_CERT_AUTH="false"
    #ETCD_TRUSTED_CA_FILE=""
    #ETCD_AUTO_TLS="false"
    #ETCD_PEER_CERT_FILE=""
    #ETCD_PEER_KEY_FILE=""
    #ETCD_PEER_CLIENT_CERT_AUTH="false"
    #ETCD_PEER_TRUSTED_CA_FILE=""
    #ETCD_PEER_AUTO_TLS="false"
    #
    #[Logging]
    #ETCD_DEBUG="false"
    #ETCD_LOG_PACKAGE_LEVELS=""
    #ETCD_LOG_OUTPUT="default"
    #
    #[Unsafe]
    #ETCD_FORCE_NEW_CLUSTER="false"
    #
    #[Version]
    #ETCD_VERSION="false"
    #ETCD_AUTO_COMPACTION_RETENTION="0"
    #
    #[Profiling]
    #ETCD_ENABLE_PPROF="false"
    #ETCD_METRICS="basic"
    #
    #[Auth]
    #ETCD_AUTH_TOKEN="simple"

### 启动etcd cluster ###

分别在3台服务器启动etcd

[root@k8s-master ~]# systemctl start etcd.service
    [root@k8s-master ~]# systemctl status etcd.service 
    ?etcd.service - Etcd Server
       Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)
       Active: active (running) since Wed 2018-01-03 09:21:36 CST; 1h 41min ago
     Main PID: 11426 (etcd)
       CGroup: /system.slice/etcd.service
               忖11426 /usr/bin/etcd --name=etcd1 --data-dir=/var/lib/etcd/default.etcd --listen-client-urls=http://10.10.200.224:2379,http://127.0.0.1:2379

查看etcd集群状态

[root@k8s-master ~]# etcdctl cluster-health
    member 359947fae86629a7 is healthy: got healthy result from http://10.10.200.224:2379
    member 4be7ddbd3bb99ca0 is healthy: got healthy result from http://10.10.200.229:2379
    member 84951a697d1bf6a0 is healthy: got healthy result from http://10.10.200.230:2379
    cluster is healthy

## Kubernetes集群配置    ##

### master节点配置 ###

1.apiserver配置文件修改，注意KUBE\_ADMISSION\_CONTROL选项的参数配置

[root@k8s-master ~]# vi /etc/kubernetes/apiserver 
    ###
    # kubernetes system config
    #
    # The following values are used to configure the kube-apiserver
    #
    
    # The address on the local server to listen to.
    #KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
    KUBE_API_ADDRESS="--address=0.0.0.0"
    
    # The port on the local server to listen on.
    KUBE_API_PORT="--port=8080"
    
    # Port minions listen on
    KUBELET_PORT="--kubelet-port=10250"
    
    # Comma separated list of nodes in the etcd cluster
    KUBE_ETCD_SERVERS="--etcd-servers=http://10.10.200.224:2379,http://10.10.200.229:2379,http://10.10.200.230:2379"
    
    # Address range to use for services
    KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
    
    # default admission control policies
    #KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
    
    KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"
    
    # Add your own!
    KUBE_API_ARGS=""

2.启动服务

[root@k8s-master ~]# systemctl start kube-apiserver
    [root@k8s-master ~]# systemctl start kube-controller-manager
    [root@k8s-master ~]# systemctl start kube-scheduler
    [root@k8s-master ~]# systemctl enable kube-apiserver
    [root@k8s-master ~]# systemctl enable kube-controller-manager
    [root@k8s-master ~]# systemctl enable kube-scheduler

### nodes节点配置 ###

1.配置config配置，node1&node2配置相同，以node1为例说明

[root@k8s-node1 ~]# vi /etc/kubernetes/config 
    ###
    # kubernetes system config
    #
    # The following values are used to configure various aspects of all
    # kubernetes services, including
    #
    #   kube-apiserver.service
    #   kube-controller-manager.service
    #   kube-scheduler.service
    #   kubelet.service
    #   kube-proxy.service
    # logging to stderr means we get it in the systemd journal
    KUBE_LOGTOSTDERR="--logtostderr=true"
    
    # journal message level, 0 is debug
    KUBE_LOG_LEVEL="--v=0"
    
    # Should this cluster be allowed to run privileged docker containers
    KUBE_ALLOW_PRIV="--allow-privileged=false"
    
    # How the controller-manager, scheduler, and proxy find the apiserver
    KUBE_MASTER="--master=http://10.10.200.224:8080"

2.配置kubelet

[root@k8s-node1 ~]# vi /etc/kubernetes/kubelet 
    ###
    # kubernetes kubelet (minion) config
    
    # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
    KUBELET_ADDRESS="--address=127.0.0.1"
    
    # The port for the info server to serve on
    # KUBELET_PORT="--port=10250"
    
    # You may leave this blank to use the actual hostname
    KUBELET_HOSTNAME="--hostname-override=10.10.200.229"
    
    # location of the api-server
    KUBELET_API_SERVER="--api-servers=http://10.10.200.224:8080"
    
    # pod infrastructure container
    KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
    
    # Add your own!
    KUBELET_ARGS=""

## 网络配置 ##

这里使用flannel进行网络配置，已经在2个节点上安装，下面进行配置。  
  
在节点上进行配置flannel

[root@k8s-node1 ~]# vi /etc/sysconfig/flanneld 
    # Flanneld configuration options
    
    # etcd url location.  Point this to the server where etcd runs
    FLANNEL_ETCD_ENDPOINTS="http://10.10.200.224:2379"
    
    # etcd config key.  This is the configuration key that flannel queries
    # For address range assignment
    FLANNEL_ETCD_PREFIX="/atomic.io/network"
    
    # Any additional options that you want to pass
    #FLANNEL_OPTIONS=""

## 查看集群状态 ##

[root@k8s-master ~]# kubectl get nodes
    NAME            STATUS    AGE
    10.10.200.229   Ready     1h
    10.10.200.230   Ready     1h
    [root@k8s-master ~]# etcdctl  member list
    359947fae86629a7: name=etcd1 peerURLs=http://10.10.200.224:2380 clientURLs=http://10.10.200.224:2379 isLeader=true
    4be7ddbd3bb99ca0: name=etcd2 peerURLs=http://10.10.200.229:2380 clientURLs=http://10.10.200.229:2379 isLeader=false
    84951a697d1bf6a0: name=etcd3 peerURLs=http://10.10.200.230:2380 clientURLs=http://10.10.200.230:2379 isLeader=false
    [root@k8s-master ~]# etcdctl cluster-health
    member 359947fae86629a7 is healthy: got healthy result from http://10.10.200.224:2379
    member 4be7ddbd3bb99ca0 is healthy: got healthy result from http://10.10.200.229:2379
    member 84951a697d1bf6a0 is healthy: got healthy result from http://10.10.200.230:2379
    cluster is healthy

# 测试使用 #

在测试使用前，现在2个节点上手动下载pod images，以及测试使用的nginx images

[root@k8s-node1 ~]# docker pull registry.access.redhat.com/rhel7/pod-infrastructure:latest
    
    [root@k8s-node1 ~]# docker pull  nginx

运行nginx

[root@k8s-master ~]# kubectl  run my-nginx --image=nginx --replicas=2 --port=80
    deployment "my-nginx" created

查看pods

[root@k8s-master ~]# kubectl get pods
    NAME                       READY     STATUS    RESTARTS   AGE
    my-nginx-379829228-55s8n   1/1       Running   0          39s
    my-nginx-379829228-th5t1   1/1       Running   0          39s

可以在node节点上查看到对应的containers

[root@k8s-node1 ~]# docker ps
    CONTAINER ID        IMAGE                                                        COMMAND                  CREATED              STATUS              PORTS               NAMES
    1c32b101cb69        nginx                                                        "nginx -g 'daemon off"   About a minute ago   Up About a minute                       k8s_my-nginx.a65fe6c_my-nginx-379829228-55s8n_default_7d5085b9-f035-11e7-a6c1-000c29183fc3_edf658c9
    de9936bea577        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           About a minute ago   Up About a minute                       k8s_POD.a8590b41_my-nginx-379829228-55s8n_default_7d5085b9-f035-11e7-a6c1-000c29183fc3_86d80c42
    
    
    [root@k8s-node2 ~]# docker ps
    CONTAINER ID        IMAGE                                                        COMMAND                  CREATED             STATUS              PORTS               NAMES
    c064a3fef6d3        nginx                                                        "nginx -g 'daemon off"   2 minutes ago       Up 2 minutes                            k8s_my-nginx.a65fe6c_my-nginx-379829228-th5t1_default_7d50757d-f035-11e7-a6c1-000c29183fc3_a2431e86
    85e437bc7eb7        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           2 minutes ago       Up 2 minutes                            k8s_POD.a8590b41_my-nginx-379829228-th5t1_default_7d50757d-f035-11e7-a6c1-000c29183fc3_419a9356

[Center]: /images/20220602/4c4693b142c34c9292367d2d5d0086cf.png