Kubernetes 高可用集群部署

集群环境介绍

Kubernetes集群是单Master的集群，对于生产环境风险太大了，非常有必要做一个高可用集群，这里的高可用主要针对控制面板来说的，比如 kube-apiserver、etcd、kube-controller-manager、kube-scheduler 这几个组件，其中 kube-controller-manager 于 kube-scheduler 组件是 Kubernetes 集群自己去实现的高可用，当有多个组件存在的时候，会自动选择一个作为 Leader 提供服务，所以不需要我们手动去实现高可用，apiserver 和 etcd 就需要手动去搭建高可用的集群的。

服务器初始化配置

关闭防火墙：
$ systemctl stop firewalld
$ systemctl disable firewalld
 
关闭selinux：
$ sed -i 's/enforcing/disabled/' /etc/selinux/config
$ setenforce 0
 
关闭swap：
$ swapoff -a  # 临时
$ sed -ri 's/.*swap.*/#&/' /etc/fstab # 永久
 
根据规划设置主机名：
hostnamectl set-hostname <hostname>
 
在Master添加Hosts：
$ cat /etc/hosts
192.168.1.101 kubernetes-master-100-1 
192.168.1.102 kubernetes-master-100-2 
192.168.1.103 kubernetes-master-100-3 
192.168.1.104 kubernetes-node-100-4 
192.168.1.105 kubernetes-node-100-5 
192.168.1.106 kubernetes-node-100-6 
192.168.1.107 kubernetes-node-100-7 
192.168.1.108 kubernetes-node-100-8 
192.168.1.109 kubernetes-node-100-9 
 
将桥接的IPv4流量传递到iptables的链：
$ cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
$ sysctl --system


cp -rf /etc/security/limits.conf /etc/security/limits.conf.back
 
cat > /etc/security/limits.conf << EOF
* soft nofile 655350
* hard nofile 655350
* soft nproc unlimited
* hard nproc unlimited
* soft core unlimited
* hard core unlimited
root soft nofile 655350
root hard nofile 655350
root soft nproc unlimited
root hard nproc unlimited
root soft core unlimited
root hard core unlimited
EOF
 
echo '/etc/sysctl.conf 文件调优'
 
cp -rf /etc/sysctl.conf /etc/sysctl.conf.back
cat > /etc/sysctl.conf << EOF
 
vm.swappiness = 0
net.ipv4.neigh.default.gc_stale_time = 120
 
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
 
# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
 
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
 
kernel.sysrq = 1
kernel.pid_max=1000000
EOF
sysctl -p
 
 
# 加载ipvs模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
 
# 查看ipvs模块加载情况
 lsmod | grep -e ip_vs -e nf_conntrack_ipv4

所有节点部署Docker/kubeadm/kubelet

# 安装Docker
$ wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
$ yum -y install docker-ce-18.06.1.ce-3.el7
$ systemctl enable docker && systemctl start docker
$ docker --version
Docker version 18.06.1-ce, build e68fc7a

# 添加阿里云YUM软件源
$ cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubeadm，kubelet和kubectl


$ yum install -y kubelet-1.20.2 kubeadm-1.20.2 kubectl-1.20.2
$ systemctl enable kubelet

部署ETCD外部集群

# 由于使用kubeadm部署的集群，默认只有一个etc节点，所以只能二进制安装好etcd集群加入。
 
cfssl 二进制包
下载 cfssl 二进制包用于签发证书，官网地址：https://pkg.cfssl.org/，下载如下文件：
 
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
 
将 cfssl 的几个二进制包添加到 PATH 包含的目录下：
$ ls cfssl*
cfssl-certinfo_linux-amd64  cfssljson_linux-amd64  cfssl_linux-amd64
$ chmod +x cfssl*
$ cp cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
$ cp cfssljson_linux-amd64 /usr/local/bin/cfssl-json
$ cp cfssl_linux-amd64 /usr/local/bin/cfssl

ETCD 二进制包
下载 ETCD 的二进制包，官方 Github 地址：https://github.com/etcd-io/etcd/releases，下载如下文件：
wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz
 
 
签发证书 ETCD 证书 准备文件
$ pwd
/opt
mkdir -p etcd/{bin,conf,data,json_file,ssl}
touch etcd/json_file/{ca-config.json,ca-csr.json,server-csr.jso


$ tree
.
etcd
├── bin
├── conf
├── data
├── json_file
|    ├── ca-config.json
|    ├── ca-csr.json
|    └── server-csr.json
└── ssl
 
# 上述几个文件的内容分别如下（标 '*' 的文件名表示该文件内容你要根据你的环境进行修改）：

$ cd /opt/etcd/
$ cat json_file/ca-config.json：
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
 
$ json_file/ca-csr.json：
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Shanghai",
            "ST": "Shanghai"
        }
    ]
}
 
$  json_file/server-csr.json*：
{
    "CN": "etcd",
    "hosts": [
                "192.168.1.101",
                "192.168.1.102",
                "192.168.1.103"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Shanghai",
            "ST": "Shanghai"
        }
    ]
}
# 此处 hosts 列表要添加上所有 etcd 节点的 IP。


# kubernetes-master-100-1上执行生成密钥
cfssl gencert -initca json_file/ca-csr.json | cfssl-json -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=json_file/ca-config.json -profile=www json_file/server-csr.json | cfssl-json -bare server
 
 
 
 
颁发证书
这里直接执行一下 generate_cert.sh 即可，该文件会在当前目录自签一套 CA 证书并基于这个 CA 为 ETCD 签发一套证书，如下：
 
$ cfssl gencert -initca json_file/ca-csr.json | cfssl-json -bare ca -
$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=json_file/ca-config.json -profile=www json_file/server-csr.json | cfssl-json -bare server
 
 
生成文件如下：
$ ls ca* server* | xargs -n1
ca.csr # CA 证书请求
ca-key.pem # CA 私钥
ca.pem # CA 证书
server.csr # etcd 证书请求
server-key.pem # etcd 私钥
server.pem # etcd 证书
 
将私钥与证书移动到 ETCD 的证书目录：
$ cp *.pem /opt/etcd/ssl/
$ ls /opt/etcd/ssl/
ca-key.pem  ca.pem  server-key.pem  server.pem
 
 
ETCD 部署
下面操作在三台master机器中进行：
将下载好的 ETCD 二进制包分发到这几个主机并解压：
 
$ tar xf etcd-v3.4.13-linux-amd64.tar.gz
$ ls etcd-v3.4.13-linux-amd64/
Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md

#将二进制包移动到 bin 目录下：
 
$ mkdir -p /opt/etcd/bin/
$ mv etcd-v3.4.13-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

# 创建 ETCD 的配置文件，
[root@kubernetes-master-100-1 ~]# cat /opt/etcd/conf/etcd.conf
#[Member]
# 节点名称，唯一
ETCD_NAME="etcd-1"
# 数据目录
ETCD_DATA_DIR="/opt/etcd/data/default.etcd"
# 集群内部通信监听的地址
ETCD_LISTEN_PEER_URLS="https://192.168.1.101:2380"
# 与客户端通信监听的地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.101:2379"
 
#[Clustering]
# 对外公告的该节点集群内监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.101:2380"
# 对外公告的该节点客户端监听地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.101:2379"
# 集群所有成员
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.101:2380,etcd-2=https://192.168.1.102:2380,etcd-3=https://192.168.1.103:2380"
# 通信 Token（密钥），可自行修改
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
# 标识是创建新的集群，加入已有的集群设为 exist
ETCD_INITIAL_CLUSTER_STATE="new"


[root@kubernetes-master-100-2 ~]# cat /opt/etcd/conf/etcd.conf
#[Member]
# 节点名称，唯一
ETCD_NAME="etcd-2"
# 数据目录
ETCD_DATA_DIR="/opt/etcd/data/default.etcd"
# 集群内部通信监听的地址
ETCD_LISTEN_PEER_URLS="https://192.168.1.102:2380"
# 与客户端通信监听的地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.102:2379"
 
#[Clustering]
# 对外公告的该节点集群内监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.102:2380"
# 对外公告的该节点客户端监听地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.102:2379"
# 集群所有成员
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.101:2380,etcd-2=https://192.168.1.102:2380,etcd-3=https://192.168.1.103:2380"
# 通信 Token（密钥），可自行修改
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
# 标识是创建新的集群，加入已有的集群设为 exist
ETCD_INITIAL_CLUSTER_STATE="new"

[root@kubernetes-master-100-3 ~]# cat /opt/etcd/conf/etcd.conf
#[Member]
# 节点名称，唯一
ETCD_NAME="etcd-3"
# 数据目录
ETCD_DATA_DIR="/opt/etcd/data/default.etcd"
# 集群内部通信监听的地址
ETCD_LISTEN_PEER_URLS="https://192.168.1.103:2380"
# 与客户端通信监听的地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.103:2379"
 
#[Clustering]
# 对外公告的该节点集群内监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.103:2380"
# 对外公告的该节点客户端监听地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.103:2379"
# 集群所有成员
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.101:2380,etcd-2=https://192.168.1.102:2380,etcd-3=https://192.168.1.103:2380"
# 通信 Token（密钥），可自行修改
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
# 标识是创建新的集群，加入已有的集群设为 exist
ETCD_INITIAL_CLUSTER_STATE="new"

# Systemd 管理
# 创建 ETCD 的 Systemd service 文件：
[root@k8s-suzaku-master1 ~]# vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
 
[Service]
Type=notify
EnvironmentFile=/opt/etcd/conf/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
        --cert-file=/opt/etcd/ssl/server.pem \
        --key-file=/opt/etcd/ssl/server-key.pem \
        --peer-cert-file=/opt/etcd/ssl/server.pem \
        --peer-key-file=/opt/etcd/ssl/server-key.pem \
        --trusted-ca-file=/opt/etcd/ssl/ca.pem \
        --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65535
 
[Install]
WantedBy=multi-user.target

# 拷贝证书、配置文件至其他两个etc节点
[root@kubernetes-master-100-1 ~]# scp /usr/lib/systemd/system/etcd.service 192.168.1.101:/usr/lib/systemd/system/
[root@kubernetes-master-100-1 ~]# scp /usr/lib/systemd/system/etcd.service 192.168.1.102:/usr/lib/systemd/system/
[root@kubernetes-master-100-1 ~]# scp -r /opt/etcd 192.168.1.101:/opt/
[root@kubernetes-master-100-1 ~]# scp -r /opt/etcd 192.168.1.102:/opt/

# 启动检查
分别在 三台master这几个节点启动 ETCD 服务并加入开机自启：
 
$ systemctl daemon-reload
$ systemctl start etcd
$ systemctl enable etcd
检查 ETCD 集群健康状态，输出内容如下则说明 ETCD 集群正常：

[root@kubernetes-master-100-1 ~]# /opt/etcd/bin/etcdctl --endpoints="https://192.168.1.101:2379,https://192.168.1.102:2379,https://192.168.1.103:2379" --cacert=/opt/etcd/ssl/ca.pem --key=/opt/etcd/ssl/server-key.pem  --cert=/opt/etcd/ssl/server.pem  endpoint health
https://192.168.1.101:2379 is healthy: successfully committed proposal: took = 22.904006ms
https://192.168.1.103:2379 is healthy: successfully committed proposal: took = 22.253018ms
https://192.168.1.102:2379 is healthy: successfully committed proposal: took = 22.998862ms

kube-vip初始化配置

首先 获取 kube-vip 的 docker 镜像，并在 /etc/kuberentes/manifests 中设置静态 pod 的 yaml 资源清单文件，这样 Kubernetes 就会自动在每个控制平面节点上部署 kube-vip 的 pod 了。 (先在master配置启动，等集群初始化完毕后在其他Master启动kube-vip即可)

# 设置VIP地址
export VIP=192.168.1.100
export INTERFACE=ens32
ctr image pull docker.io/plndr/kube-vip:0.3.1
ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip \
/kube-vip manifest pod \
--interface $INTERFACE \
--vip $VIP \
--controlplane \
--services \
--arp \
--leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml

[root@kubernetes-master-100-1 ~]# cat /etc/kubernetes/manifests/kube-vip.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: kube-vip
  namespace: kube-system
spec:
  containers:
  - args:
    - manager
    env:
    - name: vip_arp
      value: "true"
    - name: vip_interface
      value: ens32
    - name: port
      value: "6443"
    - name: vip_cidr
      value: "32"
    - name: cp_enable
      value: "true"
    - name: cp_namespace
      value: kube-system
    - name: svc_enable
      value: "true"
    - name: vip_leaderelection
      value: "true"
    - name: vip_leaseduration
      value: "5"
    - name: vip_renewdeadline
      value: "3"
    - name: vip_retryperiod
      value: "1"
    - name: vip_address
      value: 192.168.1.100
    image: plndr/kube-vip:0.3.1
    imagePullPolicy: Always
    name: kube-vip
    resources: {}
    securityContext:
      capabilities:
        add:
        - NET_ADMIN
        - NET_RAW
        - SYS_TIME
    volumeMounts:
    - mountPath: /etc/kubernetes/admin.conf
      name: kubeconfig
  hostNetwork: true
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: kubeconfig
status: {}

kubeadm 初始化集群

# 编辑config.yaml文件
[root@kubernetes-master-100-1 ~]# cat config.yaml 
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"  #使用IPVS模式，非iptables
---
apiVersion: kubeadm.k8s.io/v1beta1  #v1beta1版本,非v1alpha版本，语法会有变化
certificatesDir: /etc/kubernetes/pki  
clusterName: kubernetes
controlPlaneEndpoint: 192.168.1.100:6443  #api server IP (VIP)地址
controllerManager: {}
dns:
  type: CoreDNS  #默认DNS：CoreDNS
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers  #国内阿里镜像
kind: ClusterConfiguration
kubernetesVersion: v1.20.2   #K8S版本
networking:
  dnsDomain: cluster.local
  serviceSubnet: 172.201.0.0/16  #SVC网络段
  podSubnet: 172.200.0.0/16     #POD网络段
apiServer:
        certSANs: 
        - 192.168.1.101
        - 192.168.1.102
        - 192.168.1.103
        extraArgs:
           etcd-cafile: /opt/etcd/ssl/ca.pem
           etcd-certfile: /opt/etcd/ssl/server.pem
           etcd-keyfile: /opt/etcd/ssl/server-key.pem
etcd:  #使用外接etcd高可用
    external:
        caFile: /opt/etcd/ssl/ca.pem
        certFile: /opt/etcd/ssl/server.pem
        keyFile: /opt/etcd/ssl/server-key.pem
        endpoints:
        - https://192.168.1.101:2379
        - https://192.168.1.102:2379
        - https://192.168.1.103:2379
        
[root@kubernetes-master-100-1 ~]# kubeadm init --config config.yaml
Your Kubernetes control-plane has initialized successfully!
 
To start using your cluster, you need to run the following as a regular user:
 
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
Alternatively, if you are the root user, you can run:
 
  export KUBECONFIG=/etc/kubernetes/admin.conf
 
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
 
You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:
 
  kubeadm join 192.168.1.100:6443--token ryqarh.y3o0vhhvgy61opxz \
    --discovery-token-ca-cert-hash sha256:41db7216c10221a59d80c2b06f4756e8125caca58d29e0c16001b54dba5a9987 \
    --control-plane
 
Then you can join any number of worker nodes by running the following on each as root:
 
kubeadm join 192.168.1.100:6443 --token ryqarh.y3o0vhhvgy61opxz \
    --discovery-token-ca-cert-hash sha256:41db7216c10221a59d80c2b06f4756e8125caca58d29e0c16001b54dba5a9987
    
# 拷贝证书文件到其他master节点
scp /etc/kubernetes/pki/ca.* 192.168.1.101:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.* 192.168.1.101:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.* 192.168.1.101:/etc/kubernetes/pki/
scp /etc/kubernetes/admin.conf 192.168.1.101:/etc/kubernetes/
 
scp /etc/kubernetes/pki/ca.* 192.168.1.102:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.* 192.168.1.102:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.* 192.168.1.102:/etc/kubernetes/pki/
scp /etc/kubernetes/admin.conf 192.168.1.102:/etc/kubernetes/

# master节点加入：
kubeadm join 192.168.1.100:6443 --token ryqarh.y3o0vhhvgy61opxz \
    --discovery-token-ca-cert-hash sha256:41db7216c10221a59d80c2b06f4756e8125caca58d29e0c16001b54dba5a9987 --control-plane
     
# node 节点加入：
kubeadm join 192.168.1.100:6443 --token ryqarh.y3o0vhhvgy61opxz \
    --discovery-token-ca-cert-hash sha256:41db7216c10221a59d80c2b06f4756e8125caca58d29e0c16001b54dba5a9987
 
# 如果忘记了master节点init之后的join命令怎么办？
kubeadm token create --print-join-command


# 安装网络插件：
wget https://docs.projectcalico.org/manifests/calico.yaml
kubectl apply -f calico.yaml

[root@kubernetes-master-100-1 ~]# kubectl get nodes
NAME                      STATUS   ROLES                  AGE   VERSION
kubernetes-master-100-1   Ready    control-plane,master   66m   v1.20.2
kubernetes-master-100-2   Ready    control-plane,master   61m   v1.20.2
kubernetes-master-100-3   Ready    control-plane,master   55m   v1.20.2
kubernetes-node-100-4     Ready    <none>                 50m   v1.20.2
kubernetes-node-100-5     Ready    <none>                 50m   v1.20.2
kubernetes-node-100-6     Ready    <none>                 50m   v1.20.2
kubernetes-node-100-7     Ready    <none>                 50m   v1.20.2
kubernetes-node-100-8     Ready    <none>                 50m   v1.20.2
kubernetes-node-100-9     Ready    <none>                 50m   v1.20.2



[root@kubernetes-master-100-1 ~]# kubectl get cs 
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                 
controller-manager   Healthy   ok                 
etcd-1               Healthy   {"health":"true"}  
etcd-0               Healthy   {"health":"true"}  
etcd-2               Healthy   {"health":"true"}  
  
PS：这里注意，如果kubectl get cs 
scheduler 和 controller-manager 异常的话，检查三台master的 yaml文件，使用kubeadm安装的集群就会出现这种问题：
[root@kubernetes-master-100-1 ~]# cd /etc/kubernetes/manifests/
[root@kubernetes-master-100-1 manifests]# ll
total 12
-rw------- 1 root root 3507 Feb 25 18:55 kube-apiserver.yaml
-rw------- 1 root root 2827 Feb 25 18:58 kube-controller-manager.yaml
-rw------- 1 root root 1410 Feb 25 19:00 kube-scheduler.yaml
 
 
其中删除 kube-controller-manager.yaml 和 kube-scheduler.yaml 配置文件中的 ---port=0 参数

[root@kubernetes-master-100-1 ~]# kubectl get pods -A
NAMESPACE     NAME                                              READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-659bd7879c-pqv4b          1/1     Running   0          52m
kube-system   calico-node-2vm7t                                 1/1     Running   0          52m
kube-system   calico-node-7fqjv                                 1/1     Running   0          52m
kube-system   calico-node-88jzz                                 1/1     Running   0          52m
kube-system   calico-node-9zq87                                 1/1     Running   0          52m
kube-system   calico-node-jw9d8                                 1/1     Running   0          52m
kube-system   calico-node-qr4jk                                 1/1     Running   0          52m
kube-system   calico-node-vcdmj                                 1/1     Running   0          52m
kube-system   calico-node-wfgxg                                 1/1     Running   0          52m
kube-system   calico-node-xprj5                                 1/1     Running   0          52m
kube-system   coredns-54d67798b7-rktxc                          1/1     Running   0          71m
kube-system   coredns-54d67798b7-x428g                          1/1     Running   0          71m
kube-system   kube-apiserver-kubernetes-master-100-1            1/1     Running   0          71m
kube-system   kube-apiserver-kubernetes-master-100-2            1/1     Running   0          65m
kube-system   kube-apiserver-kubernetes-master-100-3            1/1     Running   0          60m
kube-system   kube-controller-manager-kubernetes-master-100-1   1/1     Running   0          2m17s
kube-system   kube-controller-manager-kubernetes-master-100-2   1/1     Running   0          108s
kube-system   kube-controller-manager-kubernetes-master-100-3   0/1     Running   0          86s
kube-system   kube-proxy-69dxh                                  1/1     Running   0          55m
kube-system   kube-proxy-7kkv6                                  1/1     Running   0          71m
kube-system   kube-proxy-9vs2b                                  1/1     Running   0          65m
kube-system   kube-proxy-c9nxz                                  1/1     Running   0          60m
kube-system   kube-proxy-jsppb                                  1/1     Running   0          55m
kube-system   kube-proxy-k94f4                                  1/1     Running   0          55m
kube-system   kube-proxy-mx9nc                                  1/1     Running   0          55m
kube-system   kube-proxy-q4nsq                                  1/1     Running   0          55m
kube-system   kube-proxy-spw8l                                  1/1     Running   0          55m
kube-system   kube-scheduler-kubernetes-master-100-1            1/1     Running   0          2m27s
kube-system   kube-scheduler-kubernetes-master-100-2            1/1     Running   0          115s
kube-system   kube-scheduler-kubernetes-master-100-3            0/1     Running   0          95s
kube-system   kube-vip-kubernetes-master-100-1                  1/1     Running   0          71m
kube-system   kube-vip-kubernetes-master-100-2                  1/1     Running   0          64m
kube-system   kube-vip-kubernetes-master-100-3                  1/1     Running   0          59m

kube-vip集群配置

# copy配置文件
[root@kubernetes-master-100-1 ~]# scp /etc/kubernetes/manifests/kube-vip.yaml 192.168.1.102:/etc/kubernetes/manifests/
[root@kubernetes-master-100-1 ~]# scp /etc/kubernetes/manifests/kube-vip.yaml 192.168.1.103:/etc/kubernetes/manifests/

# 登录另外的Master查看
[root@kubernetes-master-100-2 manifests]# docker ps | grep vip
c9d50e0a260f        plndr/kube-vip                                                  "/kube-vip manager"      About an hour ago   Up About an hour                        k8s_kube-vip_kube-vip-kubernetes-master-100-2_kube-system_434ab069a53e2baf875e3d827f1788d9_0
46646c0c47ba        registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2   "/pause"                 About an hour ago   Up About an hour                        k8s_POD_kube-vip-kubernetes-master-100-2_kube-system_434ab069a53e2baf875e3d827f1788d9_0


[root@kubernetes-master-100-3 manifests]# docker ps | grep vip
5f213491d320        plndr/kube-vip                                                  "/kube-vip manager"      About an hour ago   Up About an hour                        k8s_kube-vip_kube-vip-kubernetes-master-100-3_kube-system_434ab069a53e2baf875e3d827f1788d9_0
04b7c38ae306        registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2   "/pause"                 About an hour ago   Up About an hour                        k8s_POD_kube-vip-kubernetes-master-100-3_kube-system_434ab069a53e2baf875e3d827f1788d9_0

# 检查VIP是否正常
[root@kubernetes-master-100-1 ~]# ip addr | grep -A5 1.100
    inet 192.168.1.100/32 scope global ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::410a:6f16:e8df:380a/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:60:77:19:d5 brd ff:ff:ff:ff:ff:ff
    
[root@kubernetes-master-100-2 manifests]# ip addr | grep -A5 1.100
[root@kubernetes-master-100-3 manifests]# ip addr | grep -A5 1.100

# 确保VIP只出现在一个节点上说明一切正常

验证集群高可用

如果把kubernetes-master-100-1关闭，看下VIP是否会漂移到可用Master节点上，如果能正常漂移并且集群能正常访问，则集群高可用没问题！

# 关机
[root@kubernetes-master-100-1 ~]# init 0
PolicyKit daemon disconnected from the bus.
We are no longer a registered authentication agent.

# 查看VIP已漂移到kubernetes-master-100-2
[root@kubernetes-master-100-2 ~]# ip addr | grep -A5 1.100
    inet 192.168.1.100/32 scope global ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::b359:2c72:baaf:f06/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:87:85:e0:f9 brd ff:ff:ff:ff:ff:ff
    
 # 验证集群是否可以正常访问
[root@kubernetes-master-100-2 ~]# kubectl get nodes
NAME                      STATUS   ROLES                  AGE   VERSION
kubernetes-master-100-1   Ready    control-plane,master   81m   v1.20.2
kubernetes-master-100-2   Ready    control-plane,master   75m   v1.20.2
kubernetes-master-100-3   Ready    control-plane,master   70m   v1.20.2
kubernetes-node-100-4     Ready    <none>                 65m   v1.20.2
kubernetes-node-100-5     Ready    <none>                 65m   v1.20.2
kubernetes-node-100-6     Ready    <none>                 65m   v1.20.2
kubernetes-node-100-7     Ready    <none>                 65m   v1.20.2
kubernetes-node-100-8     Ready    <none>                 65m   v1.20.2
kubernetes-node-100-9     Ready    <none>                 65m   v1.20.2