目录
一、kubeadm安装要求
服务器配置至少是2G2核的。如果不是则可以在集群初始化后面增加 --ignore-preflight-errors=NumCPU
1.准备环境
1）部署软件、系统要求
2）节点规划
IP建议采用192网段，避免与kubernetes内网冲突
二、部署k8s
1、系统优化（所有节点都做）
1）关闭swap分区
#1.一旦触发 swap，会导致系统性能急剧下降，所以一般情况下，K8S 要求关闭 swap
vim /etc/fstab
用#注释掉UUID swap分区那一行
swapoff -a
echo 'KUBELET_EXTRA_ARGS="--fail-swap-on=false"' > /etc/sysconfig/kubelet #忽略swap错误
2）关闭selinux、firewalld
sed -i 's#enforcing#disabled#g' /etc/selinux/config
setenforce 0 #临时关 闭selinux
systemctl disable firewalld #永久关闭selinux
3)修改主机名并且做域名解析
#1.修改主机名
hostnamectl set-hostname k8s-m-01
hostnamectl set-hostname k8s-n-01
hostnamectl set-hostname k8s-n-02
#2.修改hosts文件
vim /etc/hosts
192.168.15.11 k8s-m-01 m1
192.168.12.12 k8s-n-01 n1
192.168.12.13 k8s-n-02 n2
4）配置免密登录、分发公钥（主节点）
ssh-keygen -t rsa
sed -i 's/#UseDNS yes/UseDNS no/g' /etc/ssh/sshd_config
for i in m1 n1 n2;do ssh-copy-id -i .ssh/id_rsa.pub root@$i;done
5）同步集群时间
#在集群中，时间是一个很重要的概念，一旦集群当中某台机器视觉按跟集群时间不一致，可能会导致集群面临很多问题。所以，在部署集群之前，需要同步集群当中的所有机器时间
yum install ntpdate -y
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' > /etc/timezone
ntpdate time2.aliyun.com
#写入定时任务
crontab -e
*/1 * * * * ntpdate time2.aliyun.com > /dev/null 2>&1
6）配置镜像源
#1.默认情况下，centos使用的是官方yum源，所以一般情况下在国内使用时非常慢的，所以我们可以替换成国内的一些比较成熟的yum源，例如：清华大学镜像源，网易云镜像源等等。
rm -rf /ect/yum.repos.d/*
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
#2.刷新缓存
yum clean all
yum makecache
7）更新系统
yum update -y --exclud=kernel*
8）安装基础常用软件
yum install wget expect vim net-tools ntp bash-completion ipvsadm ipset jq iptables conntrack sysstat libseccomp -y
9）更新系统内核（docker对系统内核要求比较高，最好用4.4+）
#如果是centos8则不需要升级内核
wget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-5.4.136-1.el7.elrepo.x86_64.rpmwget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-devel-5.4.136-1.el7.elrepo.x86_64.rpm
10）安装系统内核
yum localinstall -y kernel-lt*
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg #调到默认启动
grubby --default-kernel #查看当前默认启动的内核
reboot #重启uname -a #查看版本
11）安装IPVS
#kubernetes中service有两种代理模式，一种是iptables，一种是ipvs。两者相比，ipvs性能高，但是如果使用，需要手动加载ipvs模块
yum install -y conntrack-tools ipvsadm ipset conntrack libseccomp
cat > /etc/sysconfig/modules/ipvs.modules <<EOF #加载IPVS模块
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in ${ipvs_modules}; do
/sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
if [ $? -eq 0 ]; then
/sbin/modprobe ${kernel_module}
fi
done
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs #给文件修改权限
12）修改内核启动参数
#内核参数优化的主要目的是使其更合适kubernetes的正常运行
vim /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp.keepaliv.probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp.max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp.max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.top_timestamps = 0
net.core.somaxconn = 16384
sysctl --system #立即生效
2.安装docker（所有节点)
docker主要是作为k8s管理得常用的容器工具之一
# 1).如果之前安装过docker，需要自行卸载
sudo yum remove docker docker-common docker-selinux docker-engine -y
# 2).初始化系统环境
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
# 3).安装yum源
wget -O /etc/yum.repos.d/docker-ce.repo https://repo.huaweicloud.com/docker-ce/linux/centos/docker-ce.repo
或者
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 4).修改源
sed -i 's+download.docker.com+repo.huaweicloud.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# 5).更新yum软件包索引
yum makecache fast
# 6).安装docker
yum install docker-ce-19.03.9 -y #指定版本安装 ce是社区版 ee是企业版
# 7).docker优化
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://8mh75mhz.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
# 8).设置开机自启
systemctl enable --now docker
# 如果安装centos7步骤若出现报错：
GPG key retrieval failed: [Errno 14] curl#6 - "Could not resolve host: download.docker.com; Unknown error"
# 原因：无法解析主机
# 解决方法1：临时添加114域名解析到配置文件（临时，且不可重启网卡）
[root@k8s-master1 ~] cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 114.114.114.114
nameserver 223.5.5.5
# 解决方法2：添加到eth0网卡配置文件内（永久，需重启网卡）
DNS1=114.114.114.114
[root@k8s-m-01 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 114.114.114.114
nameserver 8.8.8.8 #或者增加谷歌的DNS
3.配置 Kubernetes yun源(所有节点)
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
setenforce 0
yum install -y kubelet kubeadm kubectl
systemctl enable --now kubelet
4.初始化master节点（只在master节点执行）
1、查看kubernetes所需要的镜像
#先检查版本
kubectl version
# 查看官网镜像下载版本
[root@k8s ~]# kubeadm config images list
k8s.gcr.io/kube-apiserver:v1.21.3
k8s.gcr.io/kube-controller-manager:v1.21.3
k8s.gcr.io/kube-scheduler:v1.21.3
k8s.gcr.io/kube-proxy:v1.21.3
k8s.gcr.io/pause:3.4.1
k8s.gcr.io/etcd:3.4.13-0
k8s.gcr.io/coredns/coredns:v1.8.0
# 查看阿里云镜像列表
[root@k8s-m-01 ~]# kubeadm config images list --image-repository=registry.cn-shanghai.aliyuncs.com/cdank8s
registry.cn-shanghai.aliyuncs.com/cdank8s/kube-apiserver:v1.21.3
registry.cn-shanghai.aliyuncs.com/cdank8s/kube-controller-manager:v1.21.3
registry.cn-shanghai.aliyuncs.com/cdank8s/kube-scheduler:v1.21.3
registry.cn-shanghai.aliyuncs.com/cdank8s/kube-proxy:v1.21.3
registry.cn-shanghai.aliyuncs.com/cdank8s/pause:3.4.1
registry.cn-shanghai.aliyuncs.com/cdank8s/etcd:3.4.13-0
registry.cn-shanghai.aliyuncs.com/cdank8s/coredns:v1.8.0
2、部署m01主节点
# 1）、初始化master节点
# --image-repository
kubeadm init --image-repository=registry.cn-hangzhou.aliyuncs.com/k8sos --kubernetes-version=v1.21.3 --service-cidr=10.96.0.0/12 --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors=Swap
# 看清版本号
# 过程中可监控初始化日志，出现successfully即为成功！
# tailf /var/log/messages
[root@k8s-master1 ~] cat /var/log/messages | grep successfully
# Mar 24 21:02:07 k8s-master1 containerd: time="2021-03-24T21:02:07.063840628+08:00" level=info msg="containerd successfully booted in 0.181480s"
# 2、 初始化完成查看kubernetes
[root@k8s-m-01 ~]# systemctl status kubelet.service
5.配置 kubernetes 用户信息（只在master节点执行）
#1.建立用户集群权限
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
#如果使用root用户，则添加至环境变量
# 临时生效
[root@k8s-m-01 ~]# export KUBECONFIG=/etc/kubernetes/admin.conf
# 永久生效
[root@k8s-m-01 ~]# cat /etc/profile.d/kubernetes.sh
export KUBECONFIG=/etc/kubernetes/admin.conf
[root@k8s-m-01 ~]# source /etc/profile
#2.增加命令提示(所又节点执行)
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bash
6.kubernetes网络插件Flannel
kubernetes 需要使用第三方的网络插件来实现 kubernetes 的网络功能，这样一来，安装网络插件成为必要前提 第三方网络插件有多种，常用的有 flanneld、calico 和 cannel（flanneld+calico），不同的网络组件，都提供基本的网络功能，为各个 Node 节点提供 IP 网络等
#1.安装集群网络插件（flannel.yaml）(主节点)
# 方式一：下载flannel.yaml #推荐
fannel3.0版本
[root@k8s-m-01 ~]# wget http://www.mmin.xyz:81/package/k8s/flannel.yaml
fannel4.0版本
[root@k8s-m-01 ~]# http://www.mmin.xyz:81/package/k8s/flannel.4.0.yaml
# 方式二：从官网复制手动编写flannel.yaml
cat >> /root/flannel.yaml <<EOF
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unused in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni
image: registry.cn-shanghai.aliyuncs.com/mmk8s/flannel:v0.14.0
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: registry.cn-shanghai.aliyuncs.com/mmk8s/flannel:v0.14.0
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
EOF
# 2、查看镜像
[root@k8s-m-01 ~]# cat flannel.yaml | grep image
image: registry.cn-hangzhou.aliyuncs.com/alvinos/flanned:v0.13.1-rc1
image: registry.cn-hangzhou.aliyuncs.com/alvinos/flanned:v0.13.1-rc1
# 如果没有替换，必须替成阿里云镜像(执行了，就无需执行)
[root@k8s-m-01 ~]# sed -i 's/quay.io/coreos/flannel/registry.cn-hangzhou.aliyuncs.com/k8sos/flannel/g' falnnel.4.0.yaml
[root@k8s-m-01 ~]# cat flannel.4.0.yaml |grep image
image: registry.cn-hangzhou.aliyuncs.com/k8sos/flannel:v0.14.0
image: registry.cn-hangzhou.aliyuncs.com/k8sos/flannel:v0.14.0
#安装集群网络插件
kubectl apply -f flannel.yaml
# 检查集群初始化状态
[root@k8s-m-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m-01 Ready control-plane,master 84m v1.21.3
[root@k8s-m-01 ~]# kubectl get pods -o wide -n kube-system
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-978bbc4b6-48jz8 1/1 Running 0 84m 10.244.0.2 k8s-m-01 <none> <none>
coredns-978bbc4b6-bc578 1/1 Running 0 84m 10.244.0.3 k8s-m-01 <none> <none>
etcd-k8s-m-01 1/1 Running 0 84m 192.168.15.11 k8s-m-01 <none> <none>
kube-apiserver-k8s-m-01 1/1 Running 0 84m 192.168.15.11 k8s-m-01 <none> <none>
kube-controller-manager-k8s-m-01 1/1 Running 0 84m 192.168.15.11 k8s-m-01 <none> <none>
kube-flannel-ds-q72k5 1/1 Running 0 11m 192.168.15.11 k8s-m-01 <none> <none>
kube-proxy-4vpbb 1/1 Running 0 84m 192.168.15.11 k8s-m-01 <none> <none>
kube-scheduler-k8s-m-01 1/1 Running 0 84m 192.168.15.11 k8s-m-01 <none> <none>
#3.将node节点加入集群
node节点要查看token，主节点生成token可重复执行查看，不会改变
[root@k8s-m-01 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.15.11:6443 --token vzlmvl.k87uoq7orcjzco91 --discovery-token-ca-cert-hash sha256:aabd53b3883043c1d38a81d859c015c8b09f143dfe7c43338e15794cf723f347
注：将上方生成的内容（token）复制到node的两台节点上执行
#4.可以查看token值命令
[root@gdx1 ~]# kubeadm token list
kubeadm join 192.168.12.11:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
注：每生成一次token值都不一样，一次token值有效期24小时
# 5、从节点加入集群
[root@k8s-n-01 yum.repos.d]# kubeadm join 192.168.15.11:6443 --token vzlmvl.k87uoq7orcjzco91 --discovery-token-ca-cert-hash sha256:aabd53b3883043c1d38a81d859c015c8b09f143dfe7c43338e15794cf723f347
[root@k8s-n-02 yum.repos.d]# kubeadm join 192.168.15.11:6443 --token vzlmvl.k87uoq7orcjzco91 --discovery-token-ca-cert-hash sha256:aabd53b3883043c1d38a81d859c015c8b09f143dfe7c43338e15794cf723f347
#扩展：生成永久Token（node加入的时候会用到）
kubeadm token create --ttl 0 --print-join-command
`kubeadm join 192.168.233.3:6443 --token rpi151.qx3660ytx2ixq8jk --discovery-token-ca-cert-hash sha256:5cf4e801c903257b50523af245f2af16a88e78dc00be3f2acc154491ad4f32a4`#这是生成的Token,node加入时使用，此``是起到注释作用，无其他用途
7.检查集群状态（主节点）
#1.第一种方式
[root@k8s-m-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m-01 Ready control-plane,master 91m v1.21.3
k8s-n-01 Ready <none> 73s v1.21.3
k8s-n-02 Ready <none> 77s v1.21.3
注：都出现ready的状态就证明成功
#2.第二种方式
[root@k8s-m-01 ~]# kubectl get pods -o wide -n kube-system
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-978bbc4b6-48jz8 1/1 Running 0 91m 10.244.0.2 k8s-m-01 <none> <none>
coredns-978bbc4b6-bc578 1/1 Running 0 91m 10.244.0.3 k8s-m-01 <none> <none>
etcd-k8s-m-01 1/1 Running 0 91m 192.168.15.11 k8s-m-01 <none> <none>
kube-apiserver-k8s-m-01 1/1 Running 0 91m 192.168.15.11 k8s-m-01 <none> <none>
kube-controller-manager-k8s-m-01 1/1 Running 0 91m 192.168.15.11 k8s-m-01 <none> <none>
kube-flannel-ds-q72k5 1/1 Running 0 18m 192.168.15.11 k8s-m-01 <none> <none>
kube-flannel-ds-sx2g9 1/1 Running 0 105s 192.168.15.13 k8s-n-02 <none> <none>
kube-flannel-ds-tc5lq 1/1 Running 0 101s 192.168.15.12 k8s-n-01 <none> <none>
kube-proxy-4vpbb 1/1 Running 0 91m 192.168.15.11 k8s-m-01 <none> <none>
kube-proxy-f6gjf 1/1 Running 0 101s 192.168.15.12 k8s-n-01 <none> <none>
kube-proxy-q9swq 1/1 Running 0 105s 192.168.15.13 k8s-n-02 <none> <none>
kube-scheduler-k8s-m-01 1/1 Running 0 91m 192.168.15.11 k8s-m-01 <none> <none>
注：看到所有状态都是1/1就证明成功
#3.第三种方式：直接验证集群DNS
kubectl run test -it --rm --image=busybox:1.28.3
If you don't see a command prompt, try pressing enter.
/ # nslookup kubernetes #进入容器输入这行内容（有内容反馈就证明成功）
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
====================================================================================
node节点加入集群失败，状态为NotReady 解决方法
#node节点与master节点版本不一致
[root@gdx1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
gdx1 Ready control-plane,master 73m v1.20.2
gdx2 NotReady <none> 10m v1.21.0
gdx3 NotReady <none> 26s v1.21.0
解决办法：
# 默认下载是最新版本，难免出现版本不一致的问题，所以下载时指定同一版本才行
# 1.从节点删除下载版本重新指定版本格式：
yum remove kubectl kubeadm kubelet -y
yum install kubectl-1.20.2 kubeadm-1.20.2 kubelet-1.20.2 -y
#2.设置开机自启
systemctl enable --now kubelet
#3.重置nonde节点配置（因为上述已经加入过集群，会报错证书，配置文件，端口号已存在，需要格式化子节点配置）
[ERROR FileAvailable--etc-kubernetes-kubelet.conf]: /etc/kubernetes/kubelet.conf already exists
[ERROR Port-10250]: Port 10250 is in use
[ERROR FileAvailable--etc-kubernetes-pki-ca.crt]: /etc/kubernetes/pki/ca.crt already exists
[root@gdx2 ~]# kubectl reset #报错以上内容执行此命令格式化子节点
#4.从集群移除状态为notready的node节点
[root@gdx1 ~]# kubectl delete node gdx3
#5.重新将node节点加入集群，此时需注意token值是否相同，如果多次生成token值，需确认最后生成的token值
注：此处做好在master节点重新生成一次token值用来node节点加入集群使用
[root@gdx1 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.12.11:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
注：将生成结果在node节点执行
#6.将node节点重新加入集群
[root@gdx2 ~]# kubeadm join 192.168.12.11:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
如果测试成功，无需执行以下命令
node节点加入集群失败，状态为NotReady 解决方法
# 1、加入集群报错解决
1> 报错原因1
[ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-iptables]: /proc/sys/net/bridge/bridge-nf-call-iptables contents are not set to 1
# 解决方法： echo "1" >/proc/sys/net/bridge/bridge-nf-call-iptables
# 2、然后重新生成，再次测试，加入集群
[root@k8s-m-01 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
注：将生成结果在node节点执行
# 3、将node节点重新加入集群
[root@k8s-m-01 ~]# kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
2> 报错原因1
从节点加入集群可能会出现如下报错：
[ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-iptables]: /proc/sys/net/bridge/bridge-nf-call-iptables contents are not set to 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher
PS：前提安装Docker+启动，再次尝试加入节点！
# 1、报错原因：
swap没关，一旦触发 swap，会导致系统性能急剧下降，所以一般情况下，所以K8S 要求关闭 swap
# 2、解决方法：
1> 执行以下三条命令后再次执行添加到集群命令：
modprobe br_netfilter
echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
echo 1 > /proc/sys/net/ipv4/ip_forward
2> 追加 --ignore-preflight-errors=Swap 参数重新执行即可！
[root@k8s-n-1 ~] kubeadm join 192.168.15.11:6443 --token iypm65.p5nmdzzw1zifxy6c --discovery-token-ca-cert-hash sha256:8bdbe324980e3350aaa3b9cea58edf576dc0a6d937da6b7bff6dbe6a01e0b525 --ignore-preflight-errors=Swap
3> 报错原因3
# 1、报错原因：
可能是内核参数忘记优化所有节点都需优化
# 2、解决方法：
回到上面第七小节，复制粘贴优化参数即可： cat > /etc/sysctl.d/k8s.conf <<EOF
4> 报错原因4
ode节点加入集群失败，状态为NotReady
情况1：软件版本不一致
[root@k8s-m-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m-01 Ready control-plane,master 16m v1.21.2
k8s-n-01 NotReady <none> 22m v1.20.2
k8s-n-02 NotReady <none> 22m v1.20.2
# 1、原因分析：
默认下载是最新版本，难免出现版本不一致的问题，所以下载时指定同一版本才行
# 2、解决方法：
yum install kubectl-1.21.2 kubeadm-1.21.2 kubelet-1.21.2 -y # 主从节点安装指定版本格式
kubectl reset # 重新初始化
kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18 # 重新将node节点加入集群
# PS：查看token值命令
[root@k8s-m-01 ~]# kubeadm token list
kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
情况2：软件版本一致，touken值可能不对
[root@k8s-m-01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m-01 Ready control-plane,master 16m v1.21.2
k8s-n-01 NotReady <none> 22m v1.20.2
k8s-n-02 NotReady <none> 22m v1.20.2
# node节点为notready状态，加入从节点时报错：
[ERROR FileAvailable--etc-kubernetes-kubelet.conf]: /etc/kubernetes/kubelet.conf already exists
[ERROR Port-10250]: Port 10250 is in use
[ERROR FileAvailable--etc-kubernetes-pki-ca.crt]: /etc/kubernetes/pki/ca.crt already exists
# 原因分析：因为多次生成主节点的token值，导致token值加入不一致或输入错误
# 解决方法：
# 从集群移除状态为notready的node节点
[root@k8s-m-01 ~]# kubectl delete n2
# node节点重置touken值以及证书端口号等信息
[root@k8s-n-01 ~]# kubeadm reset
[root@k8s-n-02 ~]# kubeadm reset
# 主节点重新创建token
[root@k8s-m-01 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
# 将node节点重新加入集群
kubeadm join 192.168.15.111:6443 --token fm0387.iqixomz5jmsukwsi --discovery-token-ca-cert-hash sha256:d8ff83ffed5967000034d07b3da738ae4f1f0254e8417bb30c21f3ed15ac5d18
# 报错 3、STATUS 状态是Healthy
[root@k8s-m-01 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
etcd-0 Healthy {"health":"true"}
1、解决方式
[root@k8s-m-01 ~]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
#- --port=0
[root@k8s-m-01 ~]# vim /etc/kubernetes/manifests/kube-scheduler.yaml
#- --port=0
2、查看状态
[root@k8s-m-01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-m-01 Ready control-plane,master 4d3h v1.21.3
k8s-n-01 Ready <none> 4d3h v1.21.3
k8s-n-02 Ready <none> 4d3h v1.21.3
