二进制安装单master节点测试环境k8s集群
一、环境规划
1.1、实验环境规划
| K8S集群角色 | Ip | 主机名 | 安装的组件 |
|---|---|---|---|
| 控制节点 | 192.168.40.180 | k8s-master1 | apiserver、controller-manager、scheduler、etcd、docker |
| 工作节点 | 192.168.40.181 | k8s-node1 | kubelet、kube-proxy、docker、calico、cordns |
实验环境规划：
- 操作系统：centos7.6
- 配置： 4Gib内存/4vCPU/100G硬盘
- 网络：Vmware NAT模式
k8s网络环境规划：
-
k8s版本：
v1.20.7 -
Pod网段：
10.0.0.0/16 -
Service网段：
10.255.0.0/16
1.2、节点初始化
1）配置静态ip地址
# 把虚拟机或者物理机配置成静态ip地址，这样机器重新启动后ip地址也不会发生改变。以master1主机修改静态IP为例
~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=none
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.40.180 # 按实验规划修改
NETMASK=255.255.255.0
GATEWAY=192.168.40.2
DNS1=223.5.5.5
# 重启网络
~]# systemctl restart network
# 测试网络连通信
~]# ping baidu.com
PING baidu.com (39.156.69.79) 56(84) bytes of data.
64 bytes from 39.156.69.79 (39.156.69.79): icmp_seq=1 ttl=128 time=63.2 ms
64 bytes from 39.156.69.79 (39.156.69.79): icmp_seq=2 ttl=128 time=47.3 ms
2）配置主机名
~]# hostnamectl set-hostname <主机名> && bash
3）配置hosts文件
# 所有机器
cat >> /etc/hosts << EOF
192.168.40.180 k8s-master1
192.168.40.181 k8s-node1
EOF
# 测试
~]# ping k8s-master1
PING k8s-master1 (192.168.40.180) 56(84) bytes of data.
64 bytes from k8s-master1 (192.168.40.180): icmp_seq=1 ttl=64 time=0.015 ms
64 bytes from k8s-master1 (192.168.40.180): icmp_seq=2 ttl=64 time=0.047 ms
4）配置主机之间无密码登录
# 生成ssh 密钥对,一路回车，不输入密码
ssh-keygen -t rsa
# 把本地的ssh公钥文件安装到远程主机对应的账户
ssh-copy-id -i .ssh/id_rsa.pub k8s-master1
ssh-copy-id -i .ssh/id_rsa.pub k8s-node1
5）关闭firewalld防火墙
systemctl stop firewalld ; systemctl disable firewalld
6）关闭selinux
# 临时关闭
setenforce 0
# 永久关闭
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
# 查看
getenforce
7）关闭交换分区swap
#临时关闭
swapoff -a
#永久关闭：注释swap挂载，给swap开头加一下注释
sed -ri 's/.*swap.*/#&/' /etc/fstab
#注意：如果是克隆的虚拟机，需要删除UUID一行
8）修改内核参数
# 1、加载br_netfilter模块
modprobe br_netfilter
# 2、验证模块是否加载成功
lsmod |grep br_netfilter
# 3、修改内核参数
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
# 4、使刚才修改的内核参数生效
sysctl -p /etc/sysctl.d/k8s.conf
9）配置阿里云repo源
# 备份
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
# 下载新的CentOS-Base.repo
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
# 生成缓存
yum clean all && yum makecache
10）配置时间同步
# 安装ntpdate命令，
yum install ntpdate -y
# 跟网络源做同步
ntpdate cn.pool.ntp.org
# 把时间同步做成计划任务
crontab -e
* */1 * * * /usr/sbin/ntpdate cn.pool.ntp.org
# 重启crond服务
service crond restart
11）安装iptables
# 安装iptables
yum install iptables-services -y
# 禁用iptables
service iptables stop && systemctl disable iptables
# 清空防火墙规则
iptables -F
12）开启ipvs
不开启ipvs将会使用iptables进行数据包转发，但是效率低，所以官网推荐需要开通ipvs。
# 创建ipvs.modules文件
~]# vim /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in ${ipvs_modules}; do
/sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
if [ 0 -eq 0 ]; then
/sbin/modprobe ${kernel_module}
fi
done
# 执行脚本
~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs
ip_vs_ftp 13079 0
nf_nat 26787 1 ip_vs_ftp
ip_vs_sed 12519 0
ip_vs_nq 12516 0
ip_vs_sh 12688 0
ip_vs_dh 12688 0
ip_vs_lblcr 12922 0
ip_vs_lblc 12819 0
ip_vs_wrr 12697 0
ip_vs_rr 12600 0
ip_vs_wlc 12519 0
ip_vs_lc 12516 0
ip_vs 141092 22 ip_vs_dh,ip_vs_lc,ip_vs_nq,ip_vs_rr,ip_vs_sh,ip_vs_ftp,ip_vs_sed,ip_vs_wlc,ip_vs_wrr,ip_vs_lblcr,ip_vs_lblc
nf_conntrack 133387 2 ip_vs,nf_nat
libcrc32c 12644 4 xfs,ip_vs,nf_nat,nf_conntrack
13）安装基础软件包
~]# yum install -y yum-utils device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel python-devel epel-release openssh-server socat ipvsadm conntrack ntpdate telnet rsync
14）安装docker-ce
~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
~]# yum install docker-ce docker-ce-cli containerd.io -y
~]# systemctl start docker && systemctl enable docker.service && systemctl status docker
15）配置docker镜像加速器
# 注意：修改docker文件驱动为systemd，默认为cgroupfs，kubelet默认使用systemd，两者必须一致才可以
~]# tee /etc/docker/daemon.json << 'EOF'
{
"registry-mirrors":["https://rsbud4vc.mirror.aliyuncs.com","https://registry.docker-cn.com","https://docker.mirrors.ustc.edu.cn","https://dockerhub.azk8s.cn","http://hub-mirror.c.163.com","http://qtid6917.mirror.aliyuncs.com", "https://rncxm540.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
~]# systemctl daemon-reload && systemctl restart docker && systemctl status docker
二、部署etcd
etcd软件下载地址：https://github.com/etcd-io/etcd/releases/
2.1、配置etcd工作目录
# 创建配置文件目录和证书文件存放目录
[root@k8s-master1 ~]# mkdir -p /etc/etcd
[root@k8s-master1 ~]# mkdir -p /etc/etcd/ssl
2.2、安装签发证书工具cfssl
[root@k8s-master1 ~]# mkdir /data/work -p && cd /data/work
[root@k8s-master1 work]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@k8s-master1 work]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@k8s-master1 work]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
[root@k8s-master1 work]# chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
[root@k8s-master1 work]# mv cfssl_linux-amd64 /usr/local/bin/cfssl
[root@k8s-master1 work]# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
[root@k8s-master1 work]# mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo
2.3、配置ca证书
1）生成ca证书请求文件
[root@k8s-master1 work]# vim ca-csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}
],
"ca": {
"expiry": "87600h"
}
}
[root@k8s-master1 work]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
[root@k8s-master1 work]# ll
total 16
-rw-r--r-- 1 root root 997 Jul 8 13:37 ca.csr
-rw-r--r-- 1 root root 252 Jul 8 13:37 ca-csr.json
-rw------- 1 root root 1675 Jul 8 13:37 ca-key.pem
-rw-r--r-- 1 root root 1346 Jul 8 13:37 ca.pem
2）生成ca证书json文件
[root@k8s-master1 work]# vim ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
2.4、生成etcd证书
# 配置etcd证书请求，hosts的ip变成自己master节点的ip
[root@k8s-master1 work]# vim etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.40.180"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
[root@k8s-master1 work]# ls etcd*.pem
etcd-key.pem etcd.pem
2.5、部署etcd
1）上传软件并解压
# 把etcd-v3.4.13-linux-amd64.tar.gz上传到k8s-master1节点的/data/work目录下
[root@k8s-master1 work]# pwd
/data/work
[root@k8s-master1 work]# tar -xf etcd-v3.4.13-linux-amd64.tar.gz
[root@k8s-master1 work]# cp -p etcd-v3.4.13-linux-amd64/etcd* /usr/local/bin/
2）创建配置文件
[root@k8s-master1 work]# vim etcd.conf
#[Member]
ETCD_NAME="etcd1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.40.180:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.40.180:2379,http://127.0.0.1:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.40.180:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.40.180:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.40.180:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
参数说明：
ETCD_NAME：节点名称，集群中唯一
ETCD_DATA_DIR：数据目录
ETCD_LISTEN_PEER_URLS：集群通信监听地址
ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS：集群通告地址
ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
ETCD_INITIAL_CLUSTER：集群节点地址
ETCD_INITIAL_CLUSTER_TOKEN：集群Token
ETCD_INITIAL_CLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群
3）创建启动服务文件
[root@k8s-master1 work]# vim etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=-/etc/etcd/etcd.conf
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd
--cert-file=/etc/etcd/ssl/etcd.pem
--key-file=/etc/etcd/ssl/etcd-key.pem
--trusted-ca-file=/etc/etcd/ssl/ca.pem
--peer-cert-file=/etc/etcd/ssl/etcd.pem
--peer-key-file=/etc/etcd/ssl/etcd-key.pem
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
--peer-client-cert-auth
--client-cert-auth
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
4）拷贝相关文件
[root@k8s-master1 work]# cp ca*.pem /etc/etcd/ssl/
[root@k8s-master1 work]# cp etcd*.pem /etc/etcd/ssl/
[root@k8s-master1 work]# cp etcd.conf /etc/etcd/
[root@k8s-master1 work]# cp etcd.service /usr/lib/systemd/system/
[root@k8s-master1 work]# mkdir -p /var/lib/etcd/default.etcd
5）启动etcd
[root@k8s-master1 work]# systemctl daemon-reload
[root@k8s-master1 work]# systemctl enable etcd.service
[root@k8s-master1 work]# systemctl start etcd.service
[root@k8s-master1 work]# systemctl status etcd
6）查看状态
[root@k8s-master1 work]# ETCDCTL_API=3
[root@k8s-master1 work]# /usr/local/bin/etcdctl --write-out=table --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.40.180:2379 endpoint health
+-----------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+-----------------------------+--------+-------------+-------+
| https://192.168.40.180:2379 | true | 27.087166ms | |
+-----------------------------+--------+-------------+-------+
三、部署kubernetes组件
二进制软件包地址： https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/
3.1、软件包下载
# 把kubernetes-server-linux-amd64.tar.gz上传到k8s-master1上的/data/work
[root@k8s-master1 work]# tar zxvf kubernetes-server-linux-amd64.tar.gz
[root@k8s-master1 work]# cd kubernetes/server/bin/
[root@k8s-master1 bin]# cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
[root@k8s-master1 bin]# scp kubelet kube-proxy k8s-node1:/usr/local/bin/
[root@k8s-master1 bin]# cd /data/work/
[root@k8s-master1 work]# mkdir -p /etc/kubernetes/
[root@k8s-master1 work]# mkdir -p /etc/kubernetes/ssl
[root@k8s-master1 work]# mkdir /var/log/kubernetes
3.2、部署api-server组件
1）创建token.csv文件
# 格式：token，用户名，UID，用户组
[root@k8s-master1 work]# cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
[root@k8s-master1 work]# cat token.csv
e4997d93ecf57d9f0b82fd2978137770,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
2）创建csr请求文件并生成证书
[root@k8s-master1 work]# vim kube-apiserver-csr.json
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.40.180",
"192.168.40.181",
"10.255.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}
]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
[root@k8s-master1 work]# ll kube-apiserver*
-rw-r--r-- 1 root root 1245 Jul 8 14:21 kube-apiserver.csr
-rw-r--r-- 1 root root 456 Jul 8 14:20 kube-apiserver-csr.json
-rw------- 1 root root 1675 Jul 8 14:21 kube-apiserver-key.pem
-rw-r--r-- 1 root root 1610 Jul 8 14:21 kube-apiserver.pem
3）创建api-server的配置文件
[root@k8s-master1 work]# vim kube-apiserver.conf
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
--anonymous-auth=false
--bind-address=192.168.40.180
--secure-port=6443
--advertise-address=192.168.40.180
--insecure-port=0
--authorization-mode=Node,RBAC
--runtime-config=api/all=true
--enable-bootstrap-token-auth
--service-cluster-ip-range=10.255.0.0/16
--token-auth-file=/etc/kubernetes/token.csv
--service-node-port-range=30000-50000
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem
--client-ca-file=/etc/kubernetes/ssl/ca.pem
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem
--service-account-issuer=https://kubernetes.default.svc.cluster.local
--etcd-cafile=/etc/etcd/ssl/ca.pem
--etcd-certfile=/etc/etcd/ssl/etcd.pem
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem
--etcd-servers=https://192.168.40.180:2379
--enable-swagger-ui=true
--allow-privileged=true
--apiserver-count=3
--audit-log-maxage=30
--audit-log-maxbackup=3
--audit-log-maxsize=100
--audit-log-path=/var/log/kube-apiserver-audit.log
--event-ttl=1h
--alsologtostderr=true
--logtostderr=false
--log-dir=/var/log/kubernetes
--v=4"
配置文件参数说明：
--logtostderr：启用日志
--v：日志等级
--log-dir：日志目录
--etcd-servers：etcd集群地址
--bind-address：监听地址
--secure-port：https安全端口
--advertise-address：集群通告地址
--allow-privileged：启用授权
--service-cluster-ip-range：Service虚拟IP地址段
--enable-admission-plugins：准入控制模块
--authorization-mode：认证授权，启用RBAC授权和节点自管理
--enable-bootstrap-token-auth：启用TLS bootstrap机制
--token-auth-file：bootstrap token文件
--service-node-port-range：Service nodeport类型默认分配端口范围
--kubelet-client-xxx：apiserver访问kubelet客户端证书
--tls-xxx-file：apiserver https证书
--etcd-xxxfile：连接Etcd集群证书 –
-audit-log-xxx：审计日志
4）创建服务启动文件
[root@k8s-master1 work]# vim kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
5）拷贝相关文件
[root@k8s-master1 work]# cp ca*.pem /etc/kubernetes/ssl
[root@k8s-master1 work]# cp kube-apiserver*.pem /etc/kubernetes/ssl/
[root@k8s-master1 work]# cp token.csv /etc/kubernetes/
[root@k8s-master1 work]# cp kube-apiserver.conf /etc/kubernetes/
[root@k8s-master1 work]# cp kube-apiserver.service /usr/lib/systemd/system/
6）启动并测试
[root@k8s-master1 work]# systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver
[root@k8s-master1 work]# systemctl status kube-apiserver
# 以下结果是正常的
[root@k8s-master1 work]# curl --insecure https://192.168.40.180:6443/
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}
3.3、部署kubectl组件
1）创建csr请求文件并生成证书
[root@k8s-master1 work]# vim admin-csr.json
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "system:masters",
"OU": "system"
}
]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
[root@k8s-master1 work]# cp admin*.pem /etc/kubernetes/ssl/
2）创建kubeconfig配置文件
kubeconfig 为 kubectl 的配置文件，包含访问 apiserver 的所有信息，如 apiserver 地址、CA 证书和自身使用的证书
# 1.设置集群参数
[root@k8s-master1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.40.180:6443 --kubeconfig=kube.config
# 2.设置客户端认证参数
[root@k8s-master1 work]# kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config
# 3.设置上下文参数
[root@k8s-master1 work]# kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config
# 4.设置默认上下文
[root@k8s-master1 work]# kubectl config use-context kubernetes --kubeconfig=kube.config
[root@k8s-master1 work]# mkdir ~/.kube -p
[root@k8s-master1 work]# cp kube.config ~/.kube/config
# 5.授权kubernetes证书访问kubelet api权限
[root@k8s-master1 work]# kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes
3）查看集群组件状态
[root@k8s-master1 work]# kubectl cluster-info
Kubernetes control plane is running at https://192.168.40.180:6443
[root@k8s-master1 work]# kubectl get componentstatuses
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
etcd-0 Healthy {"health":"true"}
[root@k8s-master1 work]# kubectl get all --all-namespaces
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.255.0.1 <none> 443/TCP
4）配置kubectl子命令补全
[root@k8s-master1 work]# yum install bash-completion -y
[root@k8s-master1 work]# kubectl completion bash > /etc/bash_completion.d/kubectl
3.4、部署kube-controller-manager组件
1）创建csr请求文件并生成证书
[root@k8s-master1 work]# vim kube-controller-manager-csr.json
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.40.180"
],
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "system:kube-controller-manager",
"OU": "system"
}
]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
[root@k8s-master1 work]# ll kube-controller-manager*.pem
-rw------- 1 root root 1675 Jul 8 15:05 kube-controller-manager-key.pem
-rw-r--r-- 1 root root 1480 Jul 8 15:05 kube-controller-manager.pem
2）创建kube-controller-manager的kubeconfig
# 1.设置集群参数
[root@k8s-master1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.40.180:6443 --kubeconfig=kube-controller-manager.kubeconfig
# 2.设置客户端认证参数
[root@k8s-master1 work]# kubectl config set-credentials system:kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
# 3.设置上下文参数
[root@k8s-master1 work]# kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
# 4.设置默认上下文
[root@k8s-master1 work]# kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
3）创建配置文件
[root@k8s-master1 work]# vim kube-controller-manager.conf
KUBE_CONTROLLER_MANAGER_OPTS="--port=0
--secure-port=10252
--bind-address=127.0.0.1
--kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig
--service-cluster-ip-range=10.255.0.0/16
--cluster-name=kubernetes
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem
--allocate-node-cidrs=true
--cluster-cidr=10.0.0.0/16
--experimental-cluster-signing-duration=87600h
--root-ca-file=/etc/kubernetes/ssl/ca.pem
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem
--leader-elect=true
--feature-gates=RotateKubeletServerCertificate=true
--controllers=*,bootstrapsigner,tokencleaner
--horizontal-pod-autoscaler-use-rest-clients=true
--horizontal-pod-autoscaler-sync-period=10s
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem
--use-service-account-credentials=true
--alsologtostderr=true
--logtostderr=false
--log-dir=/var/log/kubernetes
--v=2"
4）创建启动文件
[root@k8s-master1 work]# vim kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/etc/kubernetes/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
5）启动服务
[root@k8s-master1 work]# cp kube-controller-manager*.pem /etc/kubernetes/ssl/
[root@k8s-master1 work]# cp kube-controller-manager.kubeconfig /etc/kubernetes/
[root@k8s-master1 work]# cp kube-controller-manager.conf /etc/kubernetes/
[root@k8s-master1 work]# cp kube-controller-manager.service /usr/lib/systemd/system/
[root@k8s-master1 work]# systemctl daemon-reload
[root@k8s-master1 work]# systemctl enable kube-controller-manager
[root@k8s-master1 work]# systemctl start kube-controller-manager
[root@k8s-master1 work]# systemctl status kube-controller-manager
3.5、部署kube-scheduler组件
1）创建csr请求并生成证书
[root@k8s-master1 work]# vim kube-scheduler-csr.json
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"192.168.40.180"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "system:kube-scheduler",
"OU": "system"
}
]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
[root@k8s-master1 work]# ll kube-scheduler*.pem
-rw------- 1 root root 1679 Jul 8 15:21 kube-scheduler-key.pem
-rw-r--r-- 1 root root 1456 Jul 8 15:21 kube-scheduler.pem
2）创建kube-scheduler的kubeconfig
# 1.设置集群参数
[root@k8s-master1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.40.180:6443 --kubeconfig=kube-scheduler.kubeconfig
# 2.设置客户端认证参数
[root@k8s-master1 work]# kubectl config set-credentials system:kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
# 3.设置上下文参数
[root@k8s-master1 work]# kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
# 4.设置默认上下文
[root@k8s-master1 work]# kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
3）创建配置文件
[root@k8s-master1 work]# vim kube-scheduler.conf
KUBE_SCHEDULER_OPTS="--address=127.0.0.1
--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig
--leader-elect=true
--alsologtostderr=true
--logtostderr=false
--log-dir=/var/log/kubernetes
--v=2"
4）创建服务启动文件
[root@k8s-master1 work]# vim kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/etc/kubernetes/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
5）启动服务
[root@k8s-master1 work]# cp kube-scheduler*.pem /etc/kubernetes/ssl/
[root@k8s-master1 work]# cp kube-scheduler.kubeconfig /etc/kubernetes/
[root@k8s-master1 work]# cp kube-scheduler.conf /etc/kubernetes/
[root@k8s-master1 work]# cp kube-scheduler.service /usr/lib/systemd/system/
[root@k8s-master1 work]# systemctl daemon-reload
[root@k8s-master1 work]# systemctl enable kube-scheduler
[root@k8s-master1 work]# systemctl start kube-scheduler
[root@k8s-master1 work]# systemctl status kube-scheduler
3.6、部署kubelet组件
1）创建kubelet-bootstrap.kubeconfig
[root@k8s-master1 work]# BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)
[root@k8s-master1 work]# echo $BOOTSTRAP_TOKEN
e4997d93ecf57d9f0b82fd2978137770
[root@k8s-master1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.40.180:6443 --kubeconfig=kubelet-bootstrap.kubeconfig
[root@k8s-master1 work]# kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig
[root@k8s-master1 work]# kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
[root@k8s-master1 work]# kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
[root@k8s-master1 work]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
2）创建配置文件kubelet.json
[root@k8s-master1 work]# vim kubelet.json
{
"kind": "KubeletConfiguration",
"apiVersion": "kubelet.config.k8s.io/v1beta1",
"authentication": {
"x509": {
"clientCAFile": "/etc/kubernetes/ssl/ca.pem"
},
"webhook": {
"enabled": true,
"cacheTTL": "2m0s"
},
"anonymous": {
"enabled": false
}
},
"authorization": {
"mode": "Webhook",
"webhook": {
"cacheAuthorizedTTL": "5m0s",
"cacheUnauthorizedTTL": "30s"
}
},
"address": "192.168.40.181",
"port": 10250,
"readOnlyPort": 10255,
"cgroupDriver": "systemd",
"hairpinMode": "promiscuous-bridge",
"serializeImagePulls": false,
"featureGates": {
"RotateKubeletClientCertificate": true,
"RotateKubeletServerCertificate": true
},
"clusterDomain": "cluster.local.",
"clusterDNS": ["10.255.0.2"]
}
3）创建启动文件
[root@k8s-master1 work]# vim kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet
--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig
--cert-dir=/etc/kubernetes/ssl
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig
--config=/etc/kubernetes/kubelet.json
--network-plugin=cni
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
--alsologtostderr=true
--logtostderr=false
--log-dir=/var/log/kubernetes
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
4）拷贝相关文件
[root@k8s-node1 ~]# mkdir /etc/kubernetes/ssl -p
[root@k8s-master1 work]# scp kubelet-bootstrap.kubeconfig kubelet.json k8s-node1:/etc/kubernetes/
[root@k8s-master1 work]# scp ca.pem k8s-node1:/etc/kubernetes/ssl/
[root@k8s-master1 work]# scp kubelet.service k8s-node1:/usr/lib/systemd/system/
5）启动服务
[root@k8s-node1 ~]# mkdir -p /var/lib/kubelet
[root@k8s-node1 ~]# mkdir -p /var/log/kubernetes
[root@k8s-node1 ~]# systemctl daemon-reload
[root@k8s-node1 ~]# systemctl enable kubelet
[root@k8s-node1 ~]# systemctl start kubelet
[root@k8s-node1 ~]# systemctl status kubelet
6）控制节点上Approve一下bootstrap请求
[root@k8s-master1 work]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-R1pT4yS6b9nHGRfgqFCuSWftfeYHyaxAPqesWoYJqKg 49s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
[root@k8s-master1 work]# kubectl certificate approve node-csr-R1pT4yS6b9nHGRfgqFCuSWftfeYHyaxAPqesWoYJqKg
certificatesigningrequest.certificates.k8s.io/node-csr-R1pT4yS6b9nHGRfgqFCuSWftfeYHyaxAPqesWoYJqKg approved
[root@k8s-master1 work]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-R1pT4yS6b9nHGRfgqFCuSWftfeYHyaxAPqesWoYJqKg 76s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
[root@k8s-master1 work]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-node1 NotReady <none> 19s v1.20.7 # 还没安装网络插件
3.7、部署kube-proxy组件
1）创建csr请求并生成证书
[root@k8s-master1 work]# vim kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}
]
}
[root@k8s-master1 work]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
[root@k8s-master1 work]# ll kube-proxy*.pem
-rw------- 1 root root 1679 Jul 8 15:59 kube-proxy-key.pem
-rw-r--r-- 1 root root 1391 Jul 8 15:59 kube-proxy.pem
2）创建kubeconfig文件
[root@k8s-master1 work]# kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.40.180:6443 --kubeconfig=kube-proxy.kubeconfig
[root@k8s-master1 work]# kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
[root@k8s-master1 work]# kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
[root@k8s-master1 work]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
3）创建kube-proxy配置文件
[root@k8s-master1 work]# vim kube-proxy.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.40.181
clientConnection:
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 192.168.40.0/24
healthzBindAddress: 192.168.40.181:10256
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.40.181:10249
mode: "ipvs"
4）创建服务启动文件
[root@k8s-master1 work]# vim kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy
--config=/etc/kubernetes/kube-proxy.yaml
--alsologtostderr=true
--logtostderr=false
--log-dir=/var/log/kubernetes
--v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
5）拷贝相关文件
[root@k8s-master1 work]# scp kube-proxy.kubeconfig kube-proxy.yaml k8s-node1:/etc/kubernetes/
[root@k8s-master1 work]# scp kube-proxy.service k8s-node1:/usr/lib/systemd/system/
6）启动服务
[root@k8s-node1 ~]# mkdir -p /var/lib/kube-proxy
[root@k8s-node1 ~]# systemctl daemon-reload
[root@k8s-node1 ~]# systemctl enable kube-proxy
[root@k8s-node1 ~]# systemctl restart kube-proxy
[root@k8s-node1 ~]# systemctl status kube-proxy
3.8、部署Calico
[root@k8s-master1 work]# cat calico.yaml
# Calico Version v3.5.3
# https://docs.projectcalico.org/v3.5/releases#v3.5.3
# This manifest includes the following component versions:
# calico/node:v3.5.3
# calico/cni:v3.5.3
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
name: calico-config
namespace: kube-system
data:
# Typha is disabled.
typha_service_name: "none"
# Configure the Calico backend to use.
calico_backend: "bird"
# Configure the MTU to use
veth_mtu: "1440"
# The CNI network configuration to install on each node. The special
# values in this config will be automatically populated.
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.0",
"plugins": [
{
"type": "calico",
"log_level": "info",
"datastore_type": "kubernetes",
"nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "host-local",
"subnet": "usePodCidr"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
}
]
}
---
# This manifest installs the calico/node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: calico-node
namespace: kube-system
labels:
k8s-app: calico-node
spec:
selector:
matchLabels:
k8s-app: calico-node
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
k8s-app: calico-node
annotations:
# This, along with the CriticalAddonsOnly toleration below,
# marks the pod as a critical add-on, ensuring it gets
# priority scheduling and that its resources are reserved
# if it ever gets evicted.
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
nodeSelector:
beta.kubernetes.io/os: linux
hostNetwork: true
tolerations:
# Make sure calico-node gets scheduled on all nodes.
- effect: NoSchedule
operator: Exists
# Mark the pod as a critical add-on for rescheduling.
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
operator: Exists
serviceAccountName: calico-node
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
terminationGracePeriodSeconds: 0
initContainers:
# This container installs the Calico CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: quay.io/calico/cni:v3.5.3 # 可以修改镜像地址
command: ["/install-cni.sh"]
env:
# Name of the CNI config file to create.
- name: CNI_CONF_NAME
value: "10-calico.conflist"
# The CNI network config to install on each node.
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: calico-config
key: cni_network_config
# Set the hostname based on the k8s node name.
- name: KUBERNETES_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# CNI MTU Config variable
- name: CNI_MTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# Prevents the container from sleeping forever.
- name: SLEEP
value: "false"
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
containers:
# Runs calico/node container on each Kubernetes node. This
# container programs network policy and routes on each
# host.
- name: calico-node
image: quay.io/calico/node:v3.5.3 # 可以修改镜像地址
env:
# Use Kubernetes API as the backing datastore.
- name: DATASTORE_TYPE
value: "kubernetes"
# Wait for the datastore.
- name: WAIT_FOR_DATASTORE
value: "true"
# Set based on the k8s node name.
- name: NODENAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# Choose the backend to use.
- name: CALICO_NETWORKING_BACKEND
valueFrom:
configMapKeyRef:
name: calico-config
key: calico_backend
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: "k8s,bgp"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
- name: IP_AUTODETECTION_METHOD
value: "can-reach=192.168.40.181"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
# Set MTU for tunnel device used if ipip is enabled
- name: FELIX_IPINIPMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# The default IPv4 pool to create on startup if none exists. Pod IPs will be
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr`.
- name: CALICO_IPV4POOL_CIDR
value: "10.0.0.0/16"
# Disable file logging so `kubectl logs` works.
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
# Set Felix endpoint to host default action to ACCEPT.
- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
value: "ACCEPT"
# Disable IPv6 on Kubernetes.
- name: FELIX_IPV6SUPPORT
value: "false"
# Set Felix logging to "info"
- name: FELIX_LOGSEVERITYSCREEN
value: "info"
- name: FELIX_HEALTHENABLED
value: "true"
securityContext:
privileged: true
resources:
requests:
cpu: 250m
livenessProbe:
httpGet:
path: /liveness
port: 9099
host: localhost
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
readinessProbe:
exec:
command:
- /bin/calico-node
- -bird-ready
- -felix-ready
periodSeconds: 10
volumeMounts:
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
volumes:
# Used by calico/node.
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: var-lib-calico
hostPath:
path: /var/lib/calico
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
# Used to install CNI.
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-node
namespace: kube-system
---
# Create all the CustomResourceDefinitions needed for
# Calico policy and networking mode.
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: felixconfigurations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: FelixConfiguration
plural: felixconfigurations
singular: felixconfiguration
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: bgppeers.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: BGPPeer
plural: bgppeers
singular: bgppeer
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: bgpconfigurations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: BGPConfiguration
plural: bgpconfigurations
singular: bgpconfiguration
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ippools.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: IPPool
plural: ippools
singular: ippool
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: hostendpoints.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: HostEndpoint
plural: hostendpoints
singular: hostendpoint
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusterinformations.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: ClusterInformation
plural: clusterinformations
singular: clusterinformation
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: globalnetworkpolicies.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: GlobalNetworkPolicy
plural: globalnetworkpolicies
singular: globalnetworkpolicy
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: globalnetworksets.crd.projectcalico.org
spec:
scope: Cluster
group: crd.projectcalico.org
version: v1
names:
kind: GlobalNetworkSet
plural: globalnetworksets
singular: globalnetworkset
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: networkpolicies.crd.projectcalico.org
spec:
scope: Namespaced
group: crd.projectcalico.org
version: v1
names:
kind: NetworkPolicy
plural: networkpolicies
singular: networkpolicy
---
# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: calico-node
rules:
# The CNI plugin needs to get pods, nodes, and namespaces.
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs:
- get
- apiGroups: [""]
resources:
- endpoints
- services
verbs:
# Used to discover service IPs for advertisement.
- watch
- list
# Used to discover Typhas.
- get
- apiGroups: [""]
resources:
- nodes/status
verbs:
# Needed for clearing NodeNetworkUnavailable flag.
- patch
# Calico stores some configuration information in node annotations.
- update
# Watch for changes to Kubernetes NetworkPolicies.
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs:
- watch
- list
# Used by Calico for policy information.
- apiGroups: [""]
resources:
- pods
- namespaces
- serviceaccounts
verbs:
- list
- watch
# The CNI plugin patches pods/status.
- apiGroups: [""]
resources:
- pods/status
verbs:
- patch
# Calico monitors various CRDs for config.
- apiGroups: ["crd.projectcalico.org"]
resources:
- globalfelixconfigs
- felixconfigurations
- bgppeers
- globalbgpconfigs
- bgpconfigurations
- ippools
- globalnetworkpolicies
- globalnetworksets
- networkpolicies
- clusterinformations
- hostendpoints
verbs:
- get
- list
- watch
# Calico must create and update some CRDs on startup.
- apiGroups: ["crd.projectcalico.org"]
resources:
- ippools
- felixconfigurations
- clusterinformations
verbs:
- create
- update
# Calico stores some configuration information on the node.
- apiGroups: [""]
resources:
- nodes
verbs:
- get
- list
- watch
# These permissions are only requried for upgrade from v2.6, and can
# be removed after upgrade or on fresh installations.
- apiGroups: ["crd.projectcalico.org"]
resources:
- bgpconfigurations
- bgppeers
verbs:
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: calico-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-node
subjects:
- kind: ServiceAccount
name: calico-node
namespace: kube-system
---
[root@k8s-master1 work]# kubectl apply -f calico.yaml
[root@k8s-master1 work]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-node-t8q8z 1/1 Running 0 25s
[root@k8s-master1 work]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-node1 Ready <none> 33m v1.20.7
注意：需要修改calico.yaml文件：
- name: IP_AUTODETECTION_METHOD
value: "can-reach=192.168.40.181" # 这个ip是k8s任何一个工作节点的ip都行
3.9、部署coredns组件
[root@k8s-master1 work]# cat coredns.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf {
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: "CoreDNS"
spec:
# replicas: not specified here:
# 1. Default is 1.
# 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
nodeSelector:
kubernetes.io/os: linux
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: k8s-app
operator: In
values: ["kube-dns"]
topologyKey: kubernetes.io/hostname
containers:
- name: coredns
image: coredns/coredns:1.7.0
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.255.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
[root@k8s-master1 ~]# kubectl apply -f coredns.yaml
[root@k8s-master1 work]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
calico-node-t8q8z 1/1 Running 0 5m35s
coredns-7bf4bd64bd-5bnj7 1/1 Running 0 9s
[root@k8s-master1 work]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.255.0.2 <none> 53/UDP,53/TCP,9153/TCP 22s
3.10、测试集群部署tomcat服务
[root@k8s-master1 work]# cat tomcat.yaml
apiVersion: v1
kind: Pod
metadata:
name: demo-pod
namespace: default
labels:
app: myapp
env: dev
spec:
containers:
- name: tomcat-pod-java
ports:
- containerPort: 8080
image: tomcat:8.5-jre8-alpine
imagePullPolicy: IfNotPresent
- name: busybox
image: busybox:latest
command:
- "/bin/sh"
- "-c"
- "sleep 3600"
[root@k8s-master1 work]# cat tomcat-service.yaml
apiVersion: v1
kind: Service
metadata:
name: tomcat
spec:
type: NodePort
ports:
- port: 8080
nodePort: 30080
selector:
app: myapp
env: dev
[root@k8s-master1 work]# kubectl apply -f tomcat.yaml
pod/demo-pod created
[root@k8s-master1 work]# kubectl apply -f tomcat-service.yaml
service/tomcat created
[root@k8s-master1 work]# kubectl get pods
NAME READY STATUS RESTARTS AGE
demo-pod 2/2 Running 0 102s
[root@k8s-master1 work]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.255.0.1 <none> 443/TCP 125m
tomcat NodePort 10.255.164.138 <none> 8080:30080/TCP 112s
浏览器访问测试：
3.11、验证coredns是否正常
# busybox要用指定的1.28版本，不能用最新版本，最新版本，nslookup会解析不到dns和ip
[root@k8s-master1 ~]# kubectl run busybox --image busybox:1.28 --restart=Never --rm -it busybox -- sh
/ # ping www.baidu.com
PING www.baidu.com (39.156.66.18): 56 data bytes
64 bytes from 39.156.66.18: seq=0 ttl=127 time=39.3 ms
#通过上面可以看到能访问网络
/ # nslookup kubernetes.default.svc.cluster.local
Server: 10.255.0.2
Address 1: 10.255.0.2 kube-dns.kube-system.svc.cluster.local
Name: kubernetes.default.svc.cluster.local
Address 1: 10.255.0.1 kubernetes.default.svc.cluster.local
/ # nslookup tomcat.default.svc.cluster.local
Server: 10.255.0.2
Address 1: 10.255.0.2 kube-dns.kube-system.svc.cluster.local
Name: tomcat.default.svc.cluster.local
Address 1: 10.255.164.138 tomcat.default.svc.cluster.local
