CentOS 7基于Containerd部署Kubernetes v1.20.5

1、主机及所需软件版本
操作系统版本：CentOS7
kubernetes版本：v1.20.5
containerd版本：1.4.4
crictl版本：v1.20.0
k8s-master: 172.21.204.110
k8s-node03: 172.21.204.113

2、主机初始化配置
# 替换CentOS 7的CentOS-Base.repo文件（用华为云的，可选）
cp -a /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak
wget -O /etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo

或者（没有yum工具的情况下用curl -o）
curl -o /etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo

# 所有节点设置/etc/hosts主机名，请根据实际情况进行配置
vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
172.21.204.110 k8s-master
172.21.204.113 k8s-node03

# 验证mac地址uuid，保证各节点mac和uuid唯一
cat /sys/class/net/ens33/address
cat /sys/class/dmi/id/product_uuid

# 关闭防火墙
systemctl stop firewalld && systemctl disable firewalld

# 永久关闭seLinux（需重启系统生效）
setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

# 关闭系统交换分区swap（k8s要求关闭swap）
swapoff -a
#注释掉/etc/fstab中的swap
vi /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0
#确认swap是否被禁用（输出空值即为禁用）
cat /proc/swaps
Filename Type Size Used Priority

# 加载所需内核模块
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 加载ipvs模块（可选项，默认为iptables模式）
kuber-proxy代理支持iptables和ipvs两种模式，如果使用ipvs模式需要在初始化集群前所有节点加载ipvs模块并安装ipset工具，Linux kernel 4.19以上的内核版本使用nf_conntrack代替nf_conntrack_ipv4
cat > /etc/modules-load.d/ipvs.conf <<EOF
# Load IPVS at boot
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
EOF

systemctl enable --now systemd-modules-load.service
#确认内核模块加载成功
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
# or
cut -f1 -d " " /proc/modules | grep -e ip_vs -e nf_conntrack_ipv4
#安装ipset、ipvsadm
yum install -y ipset ipvsadm

# 设置sysctl 参数，允许iptables检查桥接流量，这些参数在重新启动后仍然存在
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
EOF

#应用sysctl参数而无需重新启动
sudo sysctl --system

3、安装kubernetes组件
# 所有节点配置kubernetes软件源（使用阿里云镜像源）
cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubelet kubeadm kubectl组件，所有节点都需要执行（默认安装最新版本）
#默认安装最新版本
yum install -y kubelet kubeadm kubectl

#指定版本号安装
yum install -y kubelet-1.20.5-0.x86_64 kubeadm-1.20.5-0.x86_64 kubectl-1.20.5-0.x86_64

# 配置kubelet使用containerd作为容器运行时，指定cgroupDriver为systemd模式（两种方法实现）
方法一：
#配置kubelet使用containerd（所有节点都要配置cgroup-driver=systemd参数，否则node节点无法自动下载和创建pod）

cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS=--cgroup-driver=systemd
EOF

方法二：
#如果不想修改/etc/sysconfig/kubelet配置，kubeadm init必须使用yaml文件来初始化传递cgroupDriver参数，可以通过如下命令导出默认的初始化配置
kubeadm config print init-defaults > kubeadm-config.yaml

#然后根据自己的需求修改配置，比如修改imageRepository的值，kube-proxy模式为ipvs，需要注意的是由于使用containerd作为运行时，所以在初始化节点的时候需要指定cgroupDriver为systemd模式
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 172.21.204.110
bindPort: 6443
nodeRegistration:
criSocket: /run/containerd/containerd.sock
name: k8s-master
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.20.5
networking:
dnsDomain: cluster.local
podSubnet: 192.168.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

# 设置kubelet开机自启
systemctl enable --now kubelet && systemctl restart kubelet

4、安装containerd容器组件
# 所有节点配置containerd软件源（containerd组件默认在docker-ce源中）
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# 查看containerd.io可用版本
yum search containerd.io --showduplicates

# 安装最新版containerd.io
yum -y install containerd.io

# 创建containerd配置文件config.toml
mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

# 修改config.toml文件中的sandbox_image、endpoint、systemd cgroup参数
vim /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri"] 字段下的sandbox_image修改为如下
sandbox_image="registry.aliyuncs.com/google_containers/pause:3.2"

[plugins."io.containerd.grpc.v1.cri".registry]字段下的endpoint修改为如下
endpoint = ["https://registry.cn-hangzhou.aliyuncs.com"]

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]字段下的SystemdCgroup修改为如下
SystemdCgroup = true

# 或使用sed脚本一键修改config.toml文件
sed -i "s#k8s.gcr.io#registry.aliyuncs.com/google_containers#g" /etc/containerd/config.toml
sed -i '/containerd.runtimes.runc.options/a SystemdCgroup = true' /etc/containerd/config.toml
sed -i "s#https://registry-1.docker.io#https://registry.cn-hangzhou.aliyuncs.com#g" /etc/containerd/config.toml

# 重启containerd.io
systemctl daemon-reload
systemctl enable containerd && systemctl restart containerd

# 安装CRI客户端工具crictl
工具下载地址：https://github.com/kubernetes-sigs/cri-tools/releases/
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.20.0/crictl-v1.20.0-linux-amd64.tar.gz

sudo tar zxvf crictl-v1.20.0-linux-amd64.tar.gz -C /usr/local/bin

cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

#或执行以下语句添加参数
crictl config runtime-endpoint unix:/run/containerd/containerd.sock

# 验证是否可用
crictl pull nginx
crictl rmi nginx
crictl images

# 重启containerd和kubelet
systemctl daemon-reload
systemctl restart containerd && systemctl restart kubelet

# 最后确认kubelet和containerd版本
containerd --version
kubelet --version

5、在master节点启动集群
方法一：使用配置文件初始化
kubeadm init --config=kubeadm-config.yaml

方法二：使用参数初始化（指定版本、master地址、基础镜像源地址、自定义ip地址）
kubeadm init --kubernetes-version=v1.20.5 --apiserver-advertise-address=172.21.204.110 --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers --service-cidr=10.96.0.0/16 --pod-network-cidr=192.168.0.0/16

简化版初始化（使用calico网络组件）
kubeadm init --image-repository registry.aliyuncs.com/google_containers --pod-network-cidr=192.168.0.0/16

简化版初始化（使用flannel网络组件）
kubeadm init --image-repository registry.aliyuncs.com/google_containers --pod-network-cidr=10.244.0.0/16

# --image-repository:指定镜像源

# 执行初始化（这里使用calico网络组件初始化）
kubeadm init --image-repository registry.aliyuncs.com/google_containers --pod-network-cidr=192.168.0.0/16

# 启动之后，需要再执行以下，否则在使用kubectl的时候会报8080端口错误：
#非root用户
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

#root用户
export KUBECONFIG=/etc/kubernetes/admin.conf

6、部署calico网络组件（默认模式）
[root@k8s-master ~]# kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created
poddisruptionbudget.policy/calico-kube-controllers created

7、node节点加入
kubeadm join 172.21.204.110:6443 --token 7pw2tl.x2121e9p3dgr7ktb --discovery-token-ca-cert-hash sha256:694e2924e2cf2211cefba492faf0259d0810d6afb72c5e941120b2911d8d18ab
默认方式添加node节点即可，不需要加--cri-socket /run/containerd/containerd.sock参数指定cri为containerd，kubernetes会自动寻找和加载containerd容器运行时

#查看node状态
[root@k8s-master ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane,master 3d21h v1.20.5
k8s-node03 Ready <none> 3d21h v1.20.5

# 查看pod状态
[root@k8s-master ~]# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-69496d8b75-drq6n 1/1 Running 0 3d21h
kube-system calico-node-4x5rf 1/1 Running 0 3d21h
kube-system calico-node-8bn96 1/1 Running 0 3d21h
kube-system coredns-7f89b7bc75-7pzx8 1/1 Running 0 3d21h
kube-system coredns-7f89b7bc75-8frqs 1/1 Running 0 3d21h
kube-system etcd-k8s-master 1/1 Running 0 3d21h
kube-system kube-apiserver-k8s-master 1/1 Running 0 3d21h
kube-system kube-controller-manager-k8s-master 1/1 Running 0 3d21h
kube-system kube-proxy-4hxhl 1/1 Running 0 3d20h
kube-system kube-proxy-bshbb 1/1 Running 0 3d20h
kube-system kube-scheduler-k8s-master 1/1 Running 0 3d21h

8、kube-proxy开启ipvs（可选）
#修改kube-proxy的configmap参数，设置mode值为ipvs
[root@k8s-master ~]# kubectl -n kube-system edit cm kube-proxy

#查看是否修改成功
[root@k8s-master ~]# kubectl -n kube-system get cm kube-proxy -o yaml | grep mode
mode: "ipvs"

#重启kube-proxy pod
[root@k8s-master ~]# kubectl -n kube-system delete pods -l k8s-app=kube-proxy

#确认ipvs模式是否开启成功
[root@k8s-master ~]# kubectl -n kube-system logs kube-proxy-4hxhl | grep ipvs
I0329 08:23:06.791059 1 server_others.go:258] Using ipvs Proxier.
[root@k8s-master ~]# kubectl -n kube-system logs kube-proxy-bshbb | grep ipvs
I0329 08:23:18.380155 1 server_others.go:258] Using ipvs Proxier.

日志中打印出Using ipvs Proxier，说明ipvs模式已经开启。

# 查看ipvs转发状态
[root@k8s-master ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.96.0.1:443 rr
-> 172.21.204.110:6443 Masq 1 0 0
TCP 10.96.0.10:53 rr
-> 192.168.235.193:53 Masq 1 0 0
-> 192.168.235.194:53 Masq 1 0 0
TCP 10.96.0.10:9153 rr
-> 192.168.235.193:9153 Masq 1 0 0
-> 192.168.235.194:9153 Masq 1 0 0
TCP 10.109.203.63:80 rr
-> 192.168.135.130:80 Masq 1 0 5
UDP 10.96.0.10:53 rr
-> 192.168.235.193:53 Masq 1 0 0
-> 192.168.235.194:53 Masq 1 0 0
————————————————
原文链接：

https://blog.csdn.net/shinaiqing/article/details/115391393

https://blog.csdn.net/tongzidane/article/details/114534734

https://blog.csdn.net/tongzidane/article/details/113942942