二进制部署k8s

主机配置

[root@localhost ~]# hostname k8s-master
[root@localhost ~]# bash
[root@k8s-master ~]#
[root@localhost ~]# hostname k8s-node01
[root@localhost ~]# bash
[root@k8s-node01 ~]#
[root@localhost ~]# hostname k8s-node02
[root@localhost ~]# bash
[root@k8s-node02 ~]#

三台主机上修改 hosts 文件添加地址解析记录

cat << EOF >> /etc/hosts
192.168.200.14 k8s-master
192.168.200.10 k8s-node01
192.168.200.11 k8s-node02
EOF
iptables -I INPUT -s 192.168.200.0/24 -j ACCEPT
sed -i '/^SELINUX=/s/enforcing/disabled/' /etc/selinux/config

生成CA证书

[root@k8s-master ~]# mkdir -p /root/software/ssl
[root@k8s-master ~]# cd /root/software/ssl/
[root@k8s-master ssl]# rz
[root@k8s-master ssl]# ls
cfssl-certinfo_linux-amd64  cfssljson_linux-amd64  cfssl_linux-amd64
[root@k8s-master ssl]# chmod +x *
[root@k8s-master ssl]# mv cfssl_linux-amd64 /usr/local/bin/cfssl
[root@k8s-master ssl]# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
[root@k8s-master ssl]# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
[root@k8s-master ssl]# cfssl --help
Usage:
Available commands:
	revoke
	serve
	genkey
	gencrl
	selfsign
	sign
	gencert
	ocsprefresh
	version
	ocspserve
	scan
	ocspsign
	info
	print-defaults
	bundle
	certinfo
	ocspdump
Top-level flags:
  -allow_verification_with_non_compliant_keys
    	Allow a SignatureVerifier to use keys which are technically non-compliant with RFC6962.
  -loglevel int
    	Log level (0 = DEBUG, 5 = FATAL) (default 1)

执行以下命令，拷贝证书生成脚本。

[root@k8s-master ssl]#  cat << EOF > ca-config.json
> {
>  "signing": {
>  "default": {
>  "expiry": "87600h"
>  },
>  "profiles": {
>  "kubernetes": {
>  "expiry": "87600h",
>  "usages": [
>  "signing",
>  "key encipherment",
>  "server auth",
>  "client auth"
>  ]
>  }
>  }
>  }
> }
> EOF

[root@k8s-master ssl]#  cat << EOF > ca-csr.json
> {
>  "CN": "kubernetes",
>  "key": {
>  "algo": "rsa",
>  "size": 2048
>  },
>  "names": [
>  {
>  "C": "CN",
>  "L": "Beijing",
>  "ST": "Beijing",
>  "O": "k8s",
>  "OU": "System"
>  }
>  ]
> }
> EOF

执行以下操作，生成 CA 证书。

[root@k8s-master ssl]#  cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2021/11/12 04:55:08 [INFO] generating a new CA key and certificate from CSR
2021/11/12 04:55:08 [INFO] generate received request
2021/11/12 04:55:08 [INFO] received CSR
2021/11/12 04:55:08 [INFO] generating key: rsa-2048
2021/11/12 04:55:09 [INFO] encoded CSR
2021/11/12 04:55:09 [INFO] signed certificate with serial number 653190800918052544492132829215453908423014945556
[root@k8s-master ssl]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem

生成server证书

[root@k8s-master ssl]#  cat << EOF > server-csr.json
> {
>  "CN": "kubernetes",
>  "hosts": [
>  "127.0.0.1",
>  "192.168.200.111",
>  "192.168.200.112",
>  "192.168.200.113",
>  "10.10.10.1",
>  "kubernetes",
>  "kubernetes.default",
>  "kubernetes.default.svc",
>  "kubernetes.default.svc.cluster",
>  "kubernetes.default.svc.cluster.local"
>  ],
>  "key": {
>  "algo": "rsa",
>  "size": 2048
>  },
>  "names": [
>  {
>  "C": "CN",
>  "L": "BeiJing",
>  cfssl gencert -initca ca-csr.json | cfssljson -bare ca -^C
[root@k8s-master ssl]#  cat << EOF > server-csr.json
{
 "CN": "kubernetes",
 "hosts": [
 "127.0.0.1",
 "192.168.200.14",
 "192.168.200.10",
 "192.168.200.11",
 "10.10.10.1",
 "kubernetes",
 "kubernetes.default",
 "kubernetes.default.svc",
 "kubernetes.default.svc.cluster",
 "kubernetes.default.svc.cluster.local"
 ],
 "key": {
 "algo": "rsa",
 "size": 2048
 },
 "names": [
 {
 "C": "CN",
 "L": "BeiJing",
 "O": "k8s",
>  "OU": "System"
>  }
>  ]
> }
> EOF
[root@k8s-master ssl]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  server-csr.json
[root@k8s-master ssl]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
2021/11/12 04:58:29 [INFO] generate received request
2021/11/12 04:58:29 [INFO] received CSR
2021/11/12 04:58:29 [INFO] generating key: rsa-2048
2021/11/12 04:58:29 [INFO] encoded CSR
2021/11/12 04:58:29 [INFO] signed certificate with serial number 85562378331608519833702058069174740363634879214
2021/11/12 04:58:29 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

生成 admin 证书

[root@k8s-master ssl]# cat << EOF > admin-csr.json
> {
>  "CN": "admin",
>  "hosts": [],
>  "key": {
>  "algo": "rsa",
>  "size": 2048
>  },
>  "names": [
>  {
>  "C": "CN",
>  "L": "BeiJing",
>  "ST": "BeiJing",
>  "O": "system:masters",
>  "OU": "System"
>  }
>  ]
> }
> 
> EOF
[root@k8s-master ssl]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -                               ^C                                                       
[root@k8s-master ssl]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2021/11/12 05:01:52 [INFO] generate received request
2021/11/12 05:01:52 [INFO] received CSR
2021/11/12 05:01:52 [INFO] generating key: rsa-2048
2021/11/12 05:01:52 [INFO] encoded CSR
2021/11/12 05:01:52 [INFO] signed certificate with serial number 643833933703109524978384066858919303910102994915
2021/11/12 05:01:52 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master ssl]# ls
admin.csr       admin.pem       ca-csr.json  server.csr       server.pem
admin-csr.json  ca-config.json  ca-key.pem   server-csr.json
admin-key.pem   ca.csr          ca.pem       server-key.pem

生成proxy证书

[root@k8s-master ssl]# cat << EOF > kube-proxy-csr.json
> {
>  "CN": "system:kube-proxy",
>  "hosts": [],
>  "key": {
>  "algo": "rsa",
>  "size": 2048
>  },
>  "names": [
>  {
>  "C": "CN",
>  "L": "BeiJing",
>  "ST": "BeiJing",
>  "O": "k8s",
>  "OU": "System"
>  }
>  ]
> }
> EOF
[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
2021/11/12 05:04:10 [INFO] generate received request
2021/11/12 05:04:10 [INFO] received CSR
2021/11/12 05:04:10 [INFO] generating key: rsa-2048
2021/11/12 05:04:10 [INFO] encoded CSR
2021/11/12 05:04:10 [INFO] signed certificate with serial number 193018966224499611445645373222754334130237086881
2021/11/12 05:04:10 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master ssl]# ls
admin.csr       ca-config.json  ca.pem               kube-proxy.pem   server.pem
admin-csr.json  ca.csr          kube-proxy.csr       server.csr
admin-key.pem   ca-csr.json     kube-proxy-csr.json  server-csr.json
admin.pem       ca-key.pem      kube-proxy-key.pem   server-key.pem

[root@k8s-master ssl]#  ls | grep -v pem | xargs -i rm {}  #删除证书以外的 json 文件，只保留 pem 证书
[root@k8s-master ssl]# ls -l
总用量 32
-rw------- 1 root root 1679 11月 12 05:01 admin-key.pem
-rw-r--r-- 1 root root 1399 11月 12 05:01 admin.pem
-rw------- 1 root root 1675 11月 12 04:55 ca-key.pem
-rw-r--r-- 1 root root 1359 11月 12 04:55 ca.pem
-rw------- 1 root root 1679 11月 12 05:04 kube-proxy-key.pem
-rw-r--r-- 1 root root 1403 11月 12 05:04 kube-proxy.pem
-rw------- 1 root root 1675 11月 12 04:58 server-key.pem
-rw-r--r-- 1 root root 1602 11月 12 04:58 server.pem

部署Etcd集群

创建文件

[root@k8s-master ssl]# mkdir /opt/kubernetes
[root@k8s-master ssl]# mkdir /opt/kubernetes/{bin,cfg,ssl}

[root@k8s-master ~]# tar xf etcd-v3.3.18-linux-amd64.tar.gz
[root@k8s-master ~]# cd etcd-v3.3.18-linux-amd64/
[root@k8s-master etcd-v3.3.18-linux-amd64]# mv etcd /opt/kubernetes/bin/
[root@k8s-master etcd-v3.3.18-linux-amd64]# mv etcdctl /opt/kubernetes/bin/

在k8s-master 上部署Etcd节点

[root@k8s-master etcd-v3.3.18-linux-amd64]# vim /opt/kubernetes/cfg/etcd
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.200.14:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.200.14:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.200.14:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.200.14:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"                             

创建脚本配置文件

[root@k8s-master etcd-v3.3.18-linux-amd64]# vim /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=-/opt/kubernetes/cfg/etcd
ExecStart=/opt/kubernetes/bin/etcd 
--name=${ETCD_NAME} 
--data-dir=${ETCD_DATA_DIR} 
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} 
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} 
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} 
--initial-cluster=${ETCD_INITIAL_CLUSTER} 
--initial-cluster-token=${ETCD_INITIAL_CLUSTER} 
--initial-cluster-state=new 
--cert-file=/opt/kubernetes/ssl/server.pem 
--key-file=/opt/kubernetes/ssl/server-key.pem 
--peer-cert-file=/opt/kubernetes/ssl/server.pem 
--peer-key-file=/opt/kubernetes/ssl/server-key.pem 
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem 
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

拷贝Etcd启动所依赖的证书

[root@k8s-master etcd-v3.3.18-linux-amd64]# cd /root/software/
[root@k8s-master software]# cp ssl/server*pem ssl/ca*.pem /opt/kubernetes/ssl/

启动Etcd主节点。若主节点卡顿，直接 ctrl +c 终止即可。实际 Etcd 进程已经启动，在连接另外两个节点时会超时，因为另外两个节点尚未启动。

[root@k8s-master software]# systemctl start etcd
^C
[root@k8s-master software]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
[root@k8s-master software]# ps aux | grep etcd
root      11025  4.1  1.8 10610508 18036 ?      Ssl  05:17   0:01 /opt/kubernetes/bin/etc --name=etcd01 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.200.14:2380 --listen-client-urls=https://192.168.200.14:2379,http://127.0.0.1:2379 --advertise-client-urls=https://192.168.200.14:2379 --initial-advertise-peer-urls=https://192.168.200.14:2380 --initial-cluster=etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380 --initial-cluster-token=etcd01=https://192.168.200.14:2380,etcd02=https://192.168.200.10:2380,etcd03=https://192.168.200.11:2380 --initial-cluster-state=new --cert-file=/opt/kubernetes/ssl/server.pem --key-file=/opt/kubernetes/ssl/server-key.pem --peer-cert-file=/opt/kubernetes/ssl/server.pem --peer-key-file=/opt/kubernetes/ssl/server-key.pem --trusted-ca-file=/opt/kubernetes/ssl/ca.pem --peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
root      11070  0.0  0.0 112728   992 pts/2    R+   05:17   0:00 grep --color=auto etcd

在node节点上部署etcd节点

[root@k8s-master ~]# rsync -avcz /opt/kubernetes/* 192.168.200.10:/opt/kubernetes/
root@192.168.200.10's password: 
sending incremental file list
created directory /opt/kubernetes
bin/
bin/etcd
bin/etcdctl
cfg/
cfg/etcd
ssl/
ssl/ca-key.pem
ssl/ca.pem
ssl/server-key.pem
ssl/server.pem

sent 13,942,496 bytes  received 199 bytes  2,535,035.45 bytes/sec
total size is 40,371,722  speedup is 2.90
[root@k8s-master ~]# rsync -avcz /opt/kubernetes/* 192.168.200.11:/opt/kubernetes/
The authenticity of host '192.168.200.11 (192.168.200.11)' can't be established.
ECDSA key fingerprint is SHA256:Ch6vBCnRcTQNR6+DnYMKJR2jvtB7y1/bB3zMjtBe3Xk.
ECDSA key fingerprint is MD5:c7:39:97:f1:e5:1f:ce:74:87:88:f2:05:83:0f:1b:5f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.200.11' (ECDSA) to the list of known hosts.
root@192.168.200.11's password: 
sending incremental file list
created directory /opt/kubernetes
bin/
bin/etcd
bin/etcdctl
cfg/
cfg/etcd
ssl/
ssl/ca-key.pem
ssl/ca.pem
ssl/server-key.pem
ssl/server.pem

sent 13,942,496 bytes  received 199 bytes  1,640,317.06 bytes/sec
total size is 40,371,722  speedup is 2.90

#修改对应下的Etcd

拷贝启动脚本

[root@k8s-master ~]# scp /usr/lib/systemd/system/etcd.service 192.168.200.10:/usr/lib/systemd/system/
root@192.168.200.10's password: 
etcd.service                                            100%  994   547.8KB/s   00:00    
[root@k8s-master ~]# scp /usr/lib/systemd/system/etcd.service 192.168.200.11:/usr/lib/systemd/system/
root@192.168.200.11's password: 
etcd.service                                            100%  994   572.8KB/s   00:00

启动node上的Etcd

[root@k8s-node01 ~]# systemctl start etcd
[root@k8s-node01 ~]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.

查看Etcd集群部署

[root@k8s-master ~]# vim /etc/profile
export PATH=$PATH:/opt/kubernetes/bin
[root@k8s-master ~]# source /etc/profile

[root@k8s-master ~]# cd /root/software/ssl/
[root@k8s-master ssl]# etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.200.14:2379,https://192.168.200.10,https://192.168.200.11:2379" cluster-health
member 7c28f3ecc1788ca9 is healthy: got healthy result from https://192.168.200.11:2379
member a1441fe9e75a6508 is healthy: got healthy result from https://192.168.200.14:2379
member ba62de56b2cc4d06 is healthy: got healthy result from https://192.168.200.10:2379
cluster is healthy

部署Flannel网络

主节点写入分配网络到Etcd，给flanneld使用

[root@k8s-master ssl]# etcdctl -ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.200.14:2379,https://192.168.200.10:2379,https://192.168.200.11:2379" set /coreos.com/network/config '{"Network":"172.17.0.0/16","Backend":{"Type":"vxlan"} }'
{"Network":"172.17.0.0/16","Backend":{"Type":"vxlan"} }

#上传flannel包，拷贝到node节点
[root@k8s-master ~]# tar xf flannel-v0.12.0-linux-amd64.tar.gz 
[root@k8s-master ~]# ls
flanneld  mk-docker-opts.sh

[root@k8s-master ~]# scp flannel mk-docker-opts.sh 192.168.200.10:/opt/kubernetes/bin/
root@192.168.200.10's password: 
flannel: No such file or directory
mk-docker-opts.sh                                    100% 2139     1.2MB/s   00:00    
[root@k8s-master ~]# scp flannel mk-docker-opts.sh 192.168.200.11:/opt/kubernetes/bin/
root@192.168.200.11's password: 
flannel: No such file or directory
mk-docker-opts.sh                                       100% 2139     1.1MB/s   00:00 

node上配置flannel

[root@k8s-node01 ~]# vim /opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcdendpoints=https://192.168.200.14:2379,https://192.168.200.10:2379,https://192.168.200.11:2379 -etcd-cafile=/opt/kubernetes/ssl/ca.pem -etcdcertfile=/opt/kubernetes/ssl/server.pem -etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"
[root@k8s-node01 ~]# scp /opt/kubernetes/cfg/flanneld 192.168.200.11:/opt/kubernetes/cfg/

在node节点上分别创建 flanneld.service 脚本文件管理 Flanneld。

cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq $FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d
/run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

在node上配置docker启动指定网段

[root@k8s-node01 ~]# vim /usr/lib/systemd/system/docker.service
EnvironmentFile=/run/flannel/subnet.env //新添加[Service]块内，目的是让 Docker 网桥分
发的 ip 地址与 flanned 网桥在同一个网段
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS //添加$
DOCKER_NETWORK_OPTIONS 变量，替换原来的 ExecStart，目的是调用 Flannel 网桥 IP
地址
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock