bgp peer
查看状态
calicoctl node status
配置全局 bgp peer(rr)
cat << EOF | calicoctl create -f -
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: bgppeer-global-3040
spec:
peerIP: 172.26.6.1
asNumber: 64567
EOF
# 删除
$ calicoctl delete bgpPeer 172.26.6.1
特定 BGP peer
$ cat << EOF | calicoctl create -f -
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: bgppeer-node-aabbff
spec:
peerIP: aa:bb::ff
node: node1
asNumber: 64514
EOF
calicoctl delete bgpPeer aa:bb::ff --scope=node --node=node1
calicoctl get bgpPeer1、 配置BGP Peer设置RR节点
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status No IPv4 peers found. IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
./calicoctl apply -f bgpconfig.yaml
root@ubuntu:~# cat bgpconfig.yaml apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: logSeverityScreen: Info nodeToNodeMeshEnabled: false asNumber: 64512
64512是全局as nu，node 默认as 是64512
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN root@ubuntu:~#
2、配置指定节点充当路由反射器
从当前节点找两个节点充当路由反射器，作为路由反射器
为方便让BGPPeer轻松选择节点，通过标签选择器匹配。给路由器反射器节点打标签：
给选定的节点指定路由反射器ID
root@ubuntu:~# ./calicoctl get node ubuntu -o yaml > rr-node.yaml
root@ubuntu:~# cat rr-node.yaml
apiVersion: projectcalico.org/v3
kind: Node
metadata:
annotations:
projectcalico.org/kube-labels: '{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"ubuntu","kubernetes.io/os":"linux","node-role.kubernetes.io/master":""}'
creationTimestamp: "2021-06-18T11:12:32Z"
labels:
beta.kubernetes.io/arch: arm64
beta.kubernetes.io/os: linux
kubernetes.io/arch: arm64
kubernetes.io/hostname: ubuntu
kubernetes.io/os: linux
node-role.kubernetes.io/master: ""
name: ubuntu
resourceVersion: "797539"
uid: 369ae2bb-756d-446d-a044-9225d9849a13
spec:
addresses:
- address: 10.10.16.82/24
type: CalicoNodeIP
- address: 10.10.16.82
type: InternalIP
bgp:
ipv4Address: 10.10.16.82/24
orchRefs:
- nodeName: ubuntu
orchestrator: k8s
status:
podCIDRs:
- 10.244.0.0/24
root@ubuntu:~#
增加 routeReflectorClusterID: 244.0.0.1
root@ubuntu:~# ./calicoctl apply -f rr-node.yaml
Successfully applied 1 'Node' resource(s)
root@ubuntu:~# cat rr-node.yaml
apiVersion: projectcalico.org/v3
kind: Node
metadata:
annotations:
projectcalico.org/kube-labels: '{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"ubuntu","kubernetes.io/os":"linux","node-role.kubernetes.io/master":""}'
creationTimestamp: "2021-06-18T11:12:32Z"
labels:
beta.kubernetes.io/arch: arm64
beta.kubernetes.io/os: linux
kubernetes.io/arch: arm64
kubernetes.io/hostname: ubuntu
kubernetes.io/os: linux
node-role.kubernetes.io/master: ""
name: ubuntu
resourceVersion: "797539"
uid: 369ae2bb-756d-446d-a044-9225d9849a13
spec:
addresses:
- address: 10.10.16.82/24
type: CalicoNodeIP
- address: 10.10.16.82
type: InternalIP
bgp:
ipv4Address: 10.10.16.82/24
routeReflectorClusterID: 244.0.0.1
orchRefs:
- nodeName: ubuntu
orchestrator: k8s
status:
podCIDRs:
- 10.244.0.0/24
3、 kubectl label node ubuntu route-reflector=true
root@ubuntu:~# kubectl get node --show-labels NAME STATUS ROLES AGE VERSION LABELS bogon Ready worker 3d21h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker cloud Ready worker 3d21h v1.21.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker ubuntu Ready master 3d21h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true root@ubuntu:~#
4、集群内RR模式
开启集群内部RR模式
此时已经选择并且配置完成节点信息，我们需要下发规则使BGP RR模式生效。
开启集群内部RR模式
#创建一个rr-rule.yaml文件，文件内容如下：
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: unbuntu #NodeName
spec:
nodeSelector: "all()"
peerSelector: "has(route-reflector)"
#应用规则开启集群内部RR模式
calicoctl create -f rr-rule.yaml
root@ubuntu:~# ./calicoctl create -f bgp-rr.yaml Successfully created 1 'BGPPeer' resource(s) root@ubuntu:~# cat bgp-rr.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: unbuntu spec: nodeSelector: all() peerSelector: route-reflector == 'true' root@ubuntu:~#
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN unbuntu all() 0
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | node specific | up | 08:45:50 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
cloud 也只有一个bpg peer
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 08:45:50 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@cloud:~#
bogon 只有一个bgp peer
You have mail in /var/spool/mail/root [root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 08:45:50 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. You have mail in /var/spool/mail/root [root@bogon ~]#
root@ubuntu:~# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES web-nginx-7bdc6b976b-7454h 1/1 Running 0 6h29m 10.244.29.4 bogon <none> <none> web-nginx-7bdc6b976b-cml8v 1/1 Running 0 6h29m 10.244.41.3 cloud <none> <none> web-nginx-7bdc6b976b-p7m2b 1/1 Running 0 6h29m 10.244.243.197 ubuntu <none> <none> root@ubuntu:~#
cloud pod
root@cloud:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default
link/ether f2:a1:42:ff:d4:86 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.244.41.3/32 brd 10.244.41.3 scope global eth0
valid_lft forever preferred_lft forever
root@cloud:~# ping 10.244.29.4
PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data.
64 bytes from 10.244.29.4: icmp_seq=1 ttl=62 time=0.269 ms
64 bytes from 10.244.29.4: icmp_seq=2 ttl=62 time=0.131 ms
^C
--- 10.244.29.4 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1013ms
rtt min/avg/max/mdev = 0.131/0.200/0.269/0.069 ms
root@cloud:~# ping 10.244.243.197
PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data.
64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.275 ms
64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.173 ms
64 bytes from 10.244.243.197: icmp_seq=3 ttl=62 time=0.128 ms
^C
--- 10.244.243.197 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2035ms
rtt min/avg/max/mdev = 0.128/0.192/0.275/0.061 ms
root@cloud:~#
bogon pod
[root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.296 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.213 ms ^C --- 10.244.243.197 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1020ms rtt min/avg/max/mdev = 0.213/0.254/0.296/0.044 ms You have mail in /var/spool/mail/root [root@bogon ~]# ping 10.244.29.4 PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data. 64 bytes from 10.244.29.4: icmp_seq=1 ttl=64 time=0.033 ms 64 bytes from 10.244.29.4: icmp_seq=2 ttl=64 time=0.039 ms ^C --- 10.244.29.4 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1061ms rtt min/avg/max/mdev = 0.033/0.036/0.039/0.003 ms [root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]#
bogon 路由
[root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.2.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 v-cali-peer 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.10.34.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i2 10.10.102.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 10.244.2.0 10.10.16.47 255.255.255.0 UG 0 0 0 enahisic2i0 10.244.29.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.29.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali2e486421e22 10.244.29.4 0.0.0.0 255.255.255.255 UH 0 0 0 calibe3388252a1 10.244.41.0 10.10.16.47 255.255.255.192 UG 0 0 0 enahisic2i0 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enahisic2i0 14.14.18.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i3.310 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 brqf1411bad-10 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 172.168.104.0 0.0.0.0 255.255.255.0 U 0 0 0 enah2i3.1022 192.168.33.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 You have mail in /var/spool/mail/root [root@bogon ~]#
cloud 路由
root@cloud:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 9.251.0.0 172.17.0.1 255.255.0.0 UG 0 0 0 docker0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.99.1.231 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.110.79.116 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.110.171.213 10.10.16.82 255.255.255.255 UGH 0 0 0 enahisic2i0 10.244.2.0 0.0.0.0 255.255.255.0 U 0 0 0 cni0 10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0 10.244.41.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.41.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali027a65c4a41 10.244.41.3 0.0.0.0 255.255.255.255 UH 0 0 0 calib81bd12045e 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enahisic2i0 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 root@cloud:~#
bogon 采用不同的as nu
root@ubuntu:~# cat bgp-peer-81.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: bogon spec: peerIP: 10.10.16.81 asNumber: 64513 root@ubuntu:~#
root@ubuntu:~# ./calicoctl apply -f bgp-peer-81.yaml Successfully applied 1 'BGPPeer' resource(s) root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+--------------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+--------------------------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | start | 09:01:58 | Active Socket: Connection | | | | | | closed | +--------------+---------------+-------+----------+--------------------------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+--------------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+--------------------------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | start | 09:01:58 | OpenSent Socket: Connection | | | | | | closed | +--------------+---------------+-------+----------+--------------------------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+--------------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+--------------------------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | start | 09:01:58 | OpenSent Socket: Connection | | | | | | closed | +--------------+---------------+-------+----------+--------------------------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-----------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-----------------------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | start | 09:03:15 | Idle BGP Error: Bad peer AS | +--------------+---------------+-------+----------+-----------------------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# cat bgp-peer-81.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: bogon spec: peerIP: 10.10.16.81 asNumber: 64513 root@ubuntu:~#
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+--------------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+--------------------------------+ | 10.10.16.82 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | start | 09:06:03 | Active BGP Error: Hold timer | | | | | | expired | +--------------+---------------+-------+----------+--------------------------------+ IPv6 BGP status No IPv6 peers found. root@cloud:~#
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+----------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+----------------------------+ | 10.10.16.82 | node specific | start | 09:05:45 | Idle Received: Bad peer AS | +--------------+---------------+-------+----------+----------------------------+ IPv6 BGP status No IPv6 peers found. You have mail in /var/spool/mail/root [root@bogon ~]#
bogon pod无法访问其他pod
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. ^C --- 10.244.243.197 ping statistics --- 1 packets transmitted, 0 received, 100% packet loss, time 0ms [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. ^C --- 10.244.41.3 ping statistics --- 2 packets transmitted, 0 received, 100% packet loss, time 1020ms [root@bogon ~]#
cloud pod 可以访问 ubuntu pod
root@cloud:~# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.271 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.145 ms ^C --- 10.244.243.197 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1017ms rtt min/avg/max/mdev = 0.145/0.208/0.271/0.063 ms root@cloud:~# ping 10.244.29.4 PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data. ^C --- 10.244.29.4 ping statistics --- 57 packets transmitted, 0 received, 100% packet loss, time 57342ms root@cloud:~# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if44: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether f2:a1:42:ff:d4:86 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.41.3/32 brd 10.244.41.3 scope global eth0 valid_lft forever preferred_lft forever root@cloud:~#
恢复bogon的as num
root@ubuntu:~# ./calicoctl apply -f bgp-peer-81.yaml Successfully applied 1 'BGPPeer' resource(s) root@ubuntu:~# cat bgp-peer-81.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: bogon spec: peerIP: 10.10.16.81 asNumber: 64512 root@ubuntu:~#
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 09:09:09 | Established | | 10.10.16.47 | node specific | up | 09:09:06 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. You have mail in /var/spool/mail/root
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | up | 09:09:08 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
竟然都有两个peer
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | global | up | 09:09:06 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@cloud:~#
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN bogon 10.10.16.81 (global) 64512 unbuntu all() 0 root@ubuntu:~# ./calicoctl get bgppeer
删除 bgpeer
root@ubuntu:~# ./calicoctl delete bgppeer bogon Successfully deleted 1 'BGPPeer' resource(s) root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN unbuntu all() 0 root@ubuntu:~# root@ubuntu:~#
bogon恢复了
root@ubuntu:~# ./calicoctl get nodes --output=wide NAME ASN IPV4 IPV6 bogon (64512) 10.10.16.81/24 cloud (64512) 10.10.16.47/24 ubuntu (64512) 10.10.16.82/24 root@ubuntu:~# ./calicoctl get nodes status resource does not exist: Node(status) with error: nodes "status" not found root@ubuntu:~# ./calicoctl get status Failed to get resources: resource type 'status' is not supported root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:50 | Established | | 10.10.16.81 | node specific | up | 09:35:24 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found.
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 09:35:24 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. You have mail in /var/spool/mail/root [root@bogon ~]#
bgpeer 恢复正常了
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 08:45:50 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@cloud:~#
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. ^C --- 10.244.243.197 ping statistics --- 1 packets transmitted, 0 received, 100% packet loss, time 0ms [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. ^C --- 10.244.41.3 ping statistics --- 2 packets transmitted, 0 received, 100% packet loss, time 1020ms [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. 64 bytes from 10.244.41.3: icmp_seq=1 ttl=62 time=0.241 ms ^C --- 10.244.41.3 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.241/0.241/0.241/0.000 ms You have mail in /var/spool/mail/root [root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.271 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.183 ms ^C --- 10.244.243.197 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1034ms rtt min/avg/max/mdev = 0.183/0.227/0.271/0.044 ms [root@bogon ~]#
为 node 节点进行分组（添加 label）
root@ubuntu:~# kubectl get node --show-labels NAME STATUS ROLES AGE VERSION LABELS bogon Ready worker 3d22h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker cloud Ready worker 3d22h v1.21.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker ubuntu Ready master 3d22h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true root@ubuntu:~#
root@ubuntu:~# kubectl label nodes ubuntu rr-id=rr1 node/ubuntu labeled root@ubuntu:~# kubectl label nodes ubuntu rr-group=rr1 node/ubuntu labeled root@ubuntu:~# vi bgp-rr.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: unbuntu spec: nodeSelector: rr-group == 'rr1' peerSelector: rr-id == 'rr1' ~
root@ubuntu:~# ./calicoctl apply -f rr1-to-node-peer.yaml Successfully applied 1 'BGPPeer' resource(s) root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:51 | Established | | 10.10.16.81 | node specific | up | 09:35:25 | Established | +--------------+---------------+-------+----------+-------------+
root@ubuntu:~# cat rr1-to-node-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr1-to-node-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-group == 'rr1' ## 通过节点选择器添加有rr-group == ‘rr1’标签的节点 peerSelector: rr-id == 'rr1' ## 通过peer选择器添加有rr-id == ‘rr1’标签的路由反射器
root@ubuntu:~# kubectl get node --show-labels NAME STATUS ROLES AGE VERSION LABELS bogon Ready worker 3d22h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker cloud Ready worker 3d22h v1.21.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker ubuntu Ready master 3d22h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true,rr-group=rr1,rr-id=rr1 root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:51 | Established | | 10.10.16.81 | node specific | up | 09:35:25 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN unbuntu all() 0 root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:45:51 | Established | | 10.10.16.81 | node specific | up | 09:35:25 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
peer仍然存在
[root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.287 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=62 time=0.189 ms ^C --- 10.244.243.197 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1021ms rtt min/avg/max/mdev = 0.189/0.238/0.287/0.049 ms You have mail in /var/spool/mail/root [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. 64 bytes from 10.244.41.3: icmp_seq=1 ttl=62 time=0.240 ms ^C --- 10.244.41.3 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.240/0.240/0.240/0.000 ms [root@bogon ~]#
原来是有两个bgp peer删掉一个就可以了
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-node-peer rr-group == 'rr1' 0 unbuntu all() 0 root@ubuntu:~# ./calicoctl delete bgppeer unbuntu Successfully deleted 1 'BGPPeer' resource(s) root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status No IPv4 peers found. IPv6 BGP status No IPv6 peers found.
给bogon 添加label
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status No IPv4 peers found. IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~# kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS bogon Ready worker 4d14h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker cloud Ready worker 4d14h v1.21.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker ubuntu Ready master 4d14h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,route-reflector=true,rr-group=rr1,rr-id=rr1 root@ubuntu:~# kubectl label nodes bogon rr-group=rr1 node/bogon labeled root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.81 | node specific | up | 01:32:08 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.81 | node specific | up | 01:32:08 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~# ./calicoctl get nodes --output=wide NAME ASN IPV4 IPV6 bogon (64512) 10.10.16.81/24 cloud (64512) 10.10.16.47/24 ubuntu (64512) 10.10.16.82/24 root@ubuntu:~#
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 01:32:07 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. You have new mail in /var/spool/mail/root [root@bogon ~]#
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status No IPv4 peers found. IPv6 BGP status No IPv6 peers found. root@cloud:~#
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 01:32:07 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. [root@bogon ~]#
bogon pod ping cloud pod 和ubuntu pod
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.286 ms ^C --- 10.244.243.197 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.286/0.286/0.286/0.000 ms [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. ^C --- 10.244.41.3 ping statistics --- 3 packets transmitted, 0 received, 100% packet loss, time 2073ms [root@bogon ~]#
leaf节点
用一条服务器模拟
[root@localhost ~]# vtysh Hello, this is Quagga (version 0.99.22.4). Copyright 1996-2005 Kunihiro Ishiguro, et al. localhost.localdomain# show running-config Building configuration... Current configuration: ! hostname localhost.localdomain hostname bgpd log stdout ! password zebra ! interface enp6s0 ipv6 nd suppress-ra ! interface enp125s0f0 ipv6 nd suppress-ra ! interface enp125s0f1 ipv6 nd suppress-ra ! interface enp125s0f2 ipv6 nd suppress-ra ! interface enp125s0f3 ipv6 nd suppress-ra ! interface lo ! router bgp 7675 bgp router-id 10.10.16.251 neighbor 10.10.16.82 remote-as 64512 ! line vty ! end localhost.localdomain# show ip bgp summary BGP router identifier 10.10.16.251, local AS number 7675 RIB entries 0, using 0 bytes of memory Peers 1, using 4560 bytes of memory Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd 10.10.16.82 4 64512 0 9 0 0 0 never Active Total number of neighbors 1 localhost.localdomain#
ubuntu配置leaf
root@ubuntu:~# cat rr1-to-leaf1-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr1-to-leaf1-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-id == 'rr1' ## 通过节点选择器添加有rr-id == 'rr1'标签的节点 peerIP: 10.10.16.251 ## leaf01交换机的地址 asNumber: 7675 ## leaf01交换机的AS号
root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml Successfully applied 1 'BGPPeer' resource(s)
root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml Successfully applied 1 'BGPPeer' resource(s)
配置完后，查看leaf
[root@localhost ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enp125s0f0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enp125s0f0 10.244.29.0 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 [root@localhost ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms 64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms ^C --- 10.244.243.197 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2054ms rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms [root@localhost ~]#
leaf节点ping bog pod
[root@localhost ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enp125s0f0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enp125s0f0 10.244.29.0 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 [root@localhost ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms 64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms ^C --- 10.244.243.197 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2054ms rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms [root@localhost ~]#
leaf节点ping ubuntu pod
[root@localhost ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enp125s0f0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enp125s0f0 10.244.29.0 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 ----要经过10.10.16.82 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enp125s0f0 [root@localhost ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=63 time=0.188 ms 64 bytes from 10.244.243.197: icmp_seq=2 ttl=63 time=0.137 ms 64 bytes from 10.244.243.197: icmp_seq=3 ttl=63 time=0.109 ms ^C --- 10.244.243.197 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2054ms rtt min/avg/max/mdev = 0.109/0.144/0.188/0.035 ms [root@localhost ~]# ping 10.244.29.4 PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data. 64 bytes from 10.244.29.4: icmp_seq=1 ttl=63 time=0.277 ms From 10.10.16.82: icmp_seq=2 Redirect Host(New nexthop: 10.10.16.81) 64 bytes from 10.244.29.4: icmp_seq=2 ttl=63 time=0.226 ms ^C --- 10.244.29.4 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1019ms rtt min/avg/max/mdev = 0.226/0.251/0.277/0.030 ms [root@localhost ~]#
对于leaf节点ping bog pod 进行tcpdump
[root@localhost ~]# ping 10.244.29.4 PING 10.244.29.4 (10.244.29.4) 56(84) bytes of data. 64 bytes from 10.244.29.4: icmp_seq=1 ttl=63 time=0.287 ms From 10.10.16.82: icmp_seq=2 Redirect Host(New nexthop: 10.10.16.81) 64 bytes from 10.244.29.4: icmp_seq=2 ttl=63 time=0.244 ms 64 bytes from 10.244.29.4: icmp_seq=3 ttl=63 time=0.134 ms ^C --- 10.244.29.4 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2075ms rtt min/avg/max/mdev = 0.134/0.221/0.287/0.066 ms [root@localhost ~]#
ubunbtu
root@ubuntu:~# tcpdump -i enahisic2i0 icmp and host 10.244.29.4 -eenv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
从enahisic2i0进，
10:49:30.999566 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
从enahisic2i0出，
10:49:30.999658 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19734, offset 0, flags [DF], proto ICMP (1), length 84)
10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64
10:49:32.034781 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64
10:49:32.034863 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64
251 mac
[root@localhost ~]# ip a sh enp125s0f0 2: enp125s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether b0:08:75:5f:b8:5b brd ff:ff:ff:ff:ff:ff inet 10.10.16.251/24 scope global enp125s0f0 valid_lft forever preferred_lft forever [root@localhost ~]#
ubuntu mac
enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:57:02:64:e7:ab brd ff:ff:ff:ff:ff:ff inet 10.10.16.82/24 brd 10.10.16.255 scope global enahisic2i0 valid_lft forever preferred_lft forever inet 10.10.16.250/32 scope global enahisic2i0 valid_lft forever preferred_lft forever inet6 fe80::4a57:2ff:fe64:e7ab/64 scope link valid_lft forever preferred_lft forever
81 mac
[root@bogon ~]# ip a sh enahisic2i0 2: enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:57:02:64:ea:1b brd ff:ff:ff:ff:ff:ff inet 10.10.16.81/24 scope global enahisic2i0 valid_lft forever preferred_lft forever inet 10.10.16.250/24 scope global secondary enahisic2i0 valid_lft forever preferred_lft forever You have mail in /var/spool/mail/root [root@bogon ~]#
bogon pod
[root@bogon ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever 2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000 link/ipip 0.0.0.0 brd 0.0.0.0 4: eth0@if103: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1440 qdisc noqueue state UP group default link/ether fa:34:76:a4:9d:f8 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.244.29.4/32 brd 10.244.29.4 scope global eth0 valid_lft forever preferred_lft forever [root@bogon ~]# ping 10.244.243.197 PING 10.244.243.197 (10.244.243.197) 56(84) bytes of data. 64 bytes from 10.244.243.197: icmp_seq=1 ttl=62 time=0.286 ms ^C --- 10.244.243.197 ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 0.286/0.286/0.286/0.000 ms [root@bogon ~]# ping 10.244.41.3 PING 10.244.41.3 (10.244.41.3) 56(84) bytes of data. ^C --- 10.244.41.3 ping statistics --- 3 packets transmitted, 0 received, 100% packet loss, time 2073ms [root@bogon ~]# tcpdump -i eth0 icmp and host 10.10.16.251 -eenv --没有snat tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 10:49:31.006872 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64 10:49:31.006906 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13073, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 1, length 64 10:49:32.042047 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64 10:49:32.042073 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13123, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 2, length 64 10:49:33.081916 ee:ee:ee:ee:ee:ee > fa:34:76:a4:9d:f8, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19767, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 3, length 64 10:49:33.081945 fa:34:76:a4:9d:f8 > ee:ee:ee:ee:ee:ee, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 13153, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4929, seq 3, length 64
251 tcpdump
[root@localhost ~]# tcpdump -i enp125s0f0 icmp and host 10.244.29.4 -eennvv tcpdump: listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes 22:57:20.268299 b0:08:75:5f:b8:5b > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 37937, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4954, seq 1, length 64 22:57:20.268462 48:57:02:64:ea:1b > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 31657, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4954, seq 1, length 64 22:57:21.322067 b0:08:75:5f:b8:5b > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 37948, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4954, seq 2, length 64 22:57:21.322198 48:57:02:64:ea:1b > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 31669, offset 0, flags [none], proto ICMP (1), length 84) 10.244.29.4 > 10.10.16.251: ICMP echo reply, id 4954, seq 2, length 64
ubuntu只有 icmp request ,reply 不经过ubuntu
root@ubuntu:~# tcpdump -i enahisic2i0 icmp and host 10.244.29.4 -eenv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 10:49:30.999566 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64 10:49:30.999658 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19734, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 1, length 64 10:49:32.034781 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64 10:49:32.034863 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 19763, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4929, seq 2, length 64 10:56:05.307169 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 32478, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 1, length 64 10:56:05.307308 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 32478, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 1, length 64 10:56:06.354681 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 32536, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 2, length 64 10:56:06.354747 48:57:02:64:e7:ab > 48:57:02:64:ea:1b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 32536, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.251 > 10.244.29.4: ICMP echo request, id 4936, seq 2, length 64 ^C
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 01:32:07 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found.
给251新增一个网段
[root@localhost ~]# ip a add 172.168.19.251/24 dev enp125s0f1 [root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enp125s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether b0:08:75:5f:b8:5b brd ff:ff:ff:ff:ff:ff inet 10.10.16.251/24 scope global enp125s0f0 valid_lft forever preferred_lft forever
ubuntu还无法访问这个网段
root@ubuntu:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0 10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7 10.244.243.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali090a0c3c4a2 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 root@ubuntu:~#
在251节点上把172.168.19.251/24这个网段发送出去
localhost.localdomain# conf t localhost.localdomain(config)# router bgp 7675 localhost.localdomain(config-router)# network 172.168.19.251/24 localhost.localdomain(config-router)# exit localhost.localdomain(config)# wr % Unknown command. localhost.localdomain(config)# exit localhost.localdomain# wr Building Configuration... Can't open configuration file /etc/quagga/zebra.conf.SbhiyG. Can't open configuration file /etc/quagga/bgpd.conf.omycEG. [OK] localhost.localdomain#
ubuntu路由信息
root@ubuntu:~# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0
10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0
10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0
10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 *
10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7
10.244.243.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali090a0c3c4a2
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
172.168.19.0 10.10.16.251 255.255.255.0 UG 0 0 0 enahisic2i0
192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
root@ubuntu:~#
ubuntu ping 172.168.19.251
root@ubuntu:~# ping 172.168.19.251 PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data. 64 bytes from 172.168.19.251: icmp_seq=1 ttl=64 time=0.124 ms 64 bytes from 172.168.19.251: icmp_seq=2 ttl=64 time=0.083 ms 64 bytes from 172.168.19.251: icmp_seq=3 ttl=64 time=0.074 ms ^C --- 172.168.19.251 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2051ms rtt min/avg/max/mdev = 0.074/0.093/0.124/0.024 ms root@ubuntu:~#
[root@localhost ~]# tcpdump -i enp125s0f0 icmp and host 172.168.19.251 -eennvv tcpdump: listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes 23:31:13.314954 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 20566, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.82 > 172.168.19.251: ICMP echo request, id 45785, seq 1, length 64 23:31:13.314964 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 23367, offset 0, flags [none], proto ICMP (1), length 84) 172.168.19.251 > 10.10.16.82: ICMP echo reply, id 45785, seq 1, length 64 23:31:14.319749 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 20813, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.82 > 172.168.19.251: ICMP echo request, id 45785, seq 2, length 64 23:31:14.319756 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 23438, offset 0, flags [none], proto ICMP (1), length 84) 172.168.19.251 > 10.10.16.82: ICMP echo reply, id 45785, seq 2, length 64
bogon 访问172.168.19.251 仍然不通，因为ubuntu没有把路由同步给bogon
[root@bogon ~]# ping 172.168.19.251 PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data. ^C --- 172.168.19.251 ping statistics --- 2 packets transmitted, 0 received, 100% packet loss, time 1056ms [root@bogon ~]#
[root@bogon ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.2.0.1 0.0.0.0 255.255.255.255 UH 0 0 0 v-cali-peer 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.10.34.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i2 10.10.102.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 10.244.29.0 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.29.1 0.0.0.0 255.255.255.255 UH 0 0 0 cali2e486421e22 10.244.29.4 0.0.0.0 255.255.255.255 UH 0 0 0 calibe3388252a1 10.244.243.192 10.10.16.82 255.255.255.192 UG 0 0 0 enahisic2i0 14.14.18.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i3.310 172.16.100.0 0.0.0.0 255.255.255.0 U 0 0 0 brqf1411bad-10 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 172.168.104.0 0.0.0.0 255.255.255.0 U 0 0 0 enah2i3.1022 192.168.33.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i1 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 You have mail in /var/spool/mail/root [root@bogon ~]# route -n | grep 172.168.19 [root@bogon ~]# tcpdump -i enahisic2i0 icmp and host 172.168.19.251 -eennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 11:31:08.028856 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 44135, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.81 > 172.168.19.251: ICMP echo request, id 0, seq 1, length 64 11:31:09.092515 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 44165, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.81 > 172.168.19.251: ICMP echo request, id 0, seq 2, length 64 ^C 2 packets captured 2 packets received by filter 0 packets dropped by kernel You have mail in /var/spool/mail/root [root@bogon ~]# tcpdump -i enahisic2i0 icmp and host 172.168.19.251 -eennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 11:33:01.475621 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 15976, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.81 > 172.168.19.251: ICMP echo request, id 45862, seq 1, length 64 11:33:02.532507 48:57:02:64:ea:1b > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 16021, offset 0, flags [DF], proto ICMP (1), length 84) 10.10.16.81 > 172.168.19.251: ICMP echo request, id 45862, seq 2, length 64 ^C 2 packets captured 2 packets received by filter 0 packets dropped by kernel [root@bogon ~]#
将251节点的as number 改成 64512
localhost.localdomain# show running-config Building configuration... Current configuration: ! hostname localhost.localdomain hostname bgpd log stdout ! password zebra ! interface enp6s0 ipv6 nd suppress-ra ! interface enp125s0f0 ipv6 nd suppress-ra ! interface enp125s0f1 ipv6 nd suppress-ra ! interface enp125s0f2 ipv6 nd suppress-ra ! interface enp125s0f3 ipv6 nd suppress-ra ! interface lo ! router bgp 7675 bgp router-id 10.10.16.251 network 172.168.19.0/24 neighbor 10.10.16.82 remote-as 64512 ! line vty ! end localhost.localdomain# conf t localhost.localdomain(config)# no router bgp 7675 localhost.localdomain(config)# router bgp 64512 localhost.localdomain(config-router)# exit localhost.localdomain(config)# wr % Unknown command. localhost.localdomain(config)# exit localhost.localdomain# wr Building Configuration... Can't open configuration file /etc/quagga/zebra.conf.ofjRYH. Can't open configuration file /etc/quagga/bgpd.conf.e0jI8H. [OK] localhost.localdomain# show running-config Building configuration... Current configuration: ! hostname localhost.localdomain hostname bgpd log stdout ! password zebra ! interface enp6s0 ipv6 nd suppress-ra ! interface enp125s0f0 ipv6 nd suppress-ra ! interface enp125s0f1 ipv6 nd suppress-ra ! interface enp125s0f2 ipv6 nd suppress-ra ! interface enp125s0f3 ipv6 nd suppress-ra ! interface lo ! router bgp 64512 bgp router-id 10.10.16.251 ! line vty ! end localhost.localdomain#
改完之后ubuntu没有172.168.19.251/24的路由了
root@ubuntu:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0 10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7 10.244.243.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali090a0c3c4a2 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.16.251 rr-id == 'rr1' 7675 rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.16.251 rr-id == 'rr1' 7675 rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~#
原因是251没有邻居了
localhost.localdomain# show ip bgp summary No IPv4 neighbor is configured localhost.localdomain#
给251 配置bgp邻居
localhost.localdomain# show ip bgp summary No IPv4 neighbor is configured localhost.localdomain# conf t localhost.localdomain(config)# router bgp 64512 localhost.localdomain(config-router)# neighbor 10.10.16.82 remote-as 64512 localhost.localdomain(config-router)# network 172.168.19.251/24 localhost.localdomain(config-router)# exit localhost.localdomain(config)# exit localhost.localdomain# wr Building Configuration... Can't open configuration file /etc/quagga/zebra.conf.9iI9n9. Can't open configuration file /etc/quagga/bgpd.conf.xLFCy9. [OK] localhost.localdomain#
更改交换机as
root@ubuntu:~# cat rr1-to-leaf1-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr1-to-leaf1-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-id == 'rr1' ## 通过节点选择器添加有rr-id == 'rr1'标签的节点 peerIP: 10.10.16.251 ## leaf01交换机的地址 asNumber: 64512 ## leaf01交换机的AS号 root@ubuntu:~#
root@ubuntu:~# ./calicoctl apply -f rr1-to-leaf1-peer.yaml Successfully applied 1 'BGPPeer' resource(s)
root@ubuntu:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.16.254 0.0.0.0 UG 0 0 0 enahisic2i0 10.10.16.0 0.0.0.0 255.255.255.0 U 0 0 0 enahisic2i0 10.244.29.0 10.10.16.81 255.255.255.192 UG 0 0 0 enahisic2i0 10.244.243.192 0.0.0.0 255.255.255.192 U 0 0 0 * 10.244.243.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali0d27bc8b0f7 10.244.243.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali090a0c3c4a2 172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 172.168.19.0 10.10.16.251 255.255.255.0 UG 0 0 0 enahisic2i0 192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 root@ubuntu:~# ping 172.168.19.251 PING 172.168.19.251 (172.168.19.251) 56(84) bytes of data. 64 bytes from 172.168.19.251: icmp_seq=1 ttl=64 time=0.292 ms 64 bytes from 172.168.19.251: icmp_seq=2 ttl=64 time=0.079 ms ^C --- 172.168.19.251 ping statistics --- 2 packets transmitted, 2 received, 0% packet loss, time 1011ms rtt min/avg/max/mdev = 0.079/0.185/0.292/0.107 ms root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.16.251 rr-id == 'rr1' 64512 rr1-to-node-peer rr-group == 'rr1' 0
bogon仍然没有172.168.19的路由
[root@bogon ~]# route -n | grep 172.168.19 [root@bogon ~]# route -n | grep 172.168.19 [root@bogon ~]#
251改造成k8s node
root@ubuntu:~# kubectl get nodes -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME bogon Ready worker 4d19h v1.18.1 10.10.16.81 <none> CentOS Linux 7 (AltArch) 4.14.0-115.8.1.el7a.aarch64 docker://20.10.7 centos7 Ready <none> 20m v1.18.1 10.10.16.251 <none> CentOS Linux 7 (AltArch) 4.14.0-115.el7a.0.1.aarch64 docker://1.13.1 cloud Ready worker 4d20h v1.21.1 10.10.16.47 <none> Ubuntu 18.04.3 LTS 5.5.19-050519-generic docker://19.3.13 ubuntu Ready master 4d20h v1.18.1 10.10.16.82 <none> Ubuntu 18.04.3 LTS 5.0.0-23-generic containerd://1.3.7 root@ubuntu:~#
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+--------------------------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+--------------------------------+ | 10.10.16.251 | node specific | start | 07:14:44 | Idle Socket: Connection | | | | | | refused | | 14.14.18.89 | node specific | start | 07:14:44 | Connect | +--------------+---------------+-------+----------+--------------------------------+
root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.16.251 rr-id == 'rr1' 64512 rr1-to-node-peer rr-group == 'rr1' 0
删掉bgp pper
root@ubuntu:~# ./calicoctl delete bgppeer rr1-to-leaf1-peer Successfully deleted 1 'BGPPeer' resource(s) root@ubuntu:~# kubectl get nodes -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME bogon Ready worker 4d20h v1.18.1 10.10.16.81 <none> CentOS Linux 7 (AltArch) 4.14.0-115.8.1.el7a.aarch64 docker://20.10.7 centos7 Ready <none> 21m v1.18.1 10.10.16.251 <none> CentOS Linux 7 (AltArch) 4.14.0-115.el7a.0.1.aarch64 docker://1.13.1 cloud Ready worker 4d20h v1.21.1 10.10.16.47 <none> Ubuntu 18.04.3 LTS 5.5.19-050519-generic docker://19.3.13 ubuntu Ready master 4d20h v1.18.1 10.10.16.82 <none> Ubuntu 18.04.3 LTS 5.0.0-23-generic containerd://1.3.7 root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-node-peer rr-group == 'rr1' 0 root@ubuntu:~#
root@ubuntu:~# kubectl get pods -o wide -n kube-system NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES calico-kube-controllers-5978c5f6b5-tk6pg 1/1 Running 0 4d20h 10.244.243.194 ubuntu <none> <none> calico-node-gnp9x 0/1 Running 0 59m 10.10.16.82 ubuntu <none> <none> calico-node-jcvsf 1/1 Running 0 59m 10.10.16.251 centos7 <none> <none> calico-node-pq756 0/1 Running 0 59m 10.10.16.81 bogon <none> <none> calico-node-pxr58 1/1 Running 0 52m 10.10.16.47 cloud <none> <none>
bird: Reconfiguration requested by SIGHUP bird: Reconfiguring bird: device1: Reconfigured bird: direct1: Reconfigured bird: Adding protocol Node_14_14_18_89 bird: Node_14_14_18_89: Initializing bird: Node_14_14_18_89: Starting bird: Node_14_14_18_89: State changed to start bird: Reconfigured 2021-06-23 08:03:04.084 [INFO][90] felix/summary.go 100: Summarising 9 dataplane reconciliation loops over 1m2.3s: avg=5ms longest=11ms (resync-nat-v4,resync-raw-v4) 2021-06-23 08:03:12.698 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:03:42.548 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:03:42.699 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:04:07.433 [INFO][90] felix/summary.go 100: Summarising 11 dataplane reconciliation loops over 1m3.3s: avg=5ms longest=16ms () 2021-06-23 08:04:12.702 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:04:42.549 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:04:42.703 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:05:09.720 [INFO][90] felix/summary.go 100: Summarising 8 dataplane reconciliation loops over 1m2.3s: avg=4ms longest=19ms () 2021-06-23 08:05:12.705 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:05:42.552 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:05:42.706 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:06:11.983 [INFO][90] felix/summary.go 100: Summarising 9 dataplane reconciliation loops over 1m2.3s: avg=4ms longest=12ms (resync-nat-v4,resync-raw-v4) 2021-06-23 08:06:12.708 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:06:36.324 [INFO][93] confd/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/ipam/v2/host/ubuntu" 2021-06-23 08:06:42.554 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:06:42.709 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:07:09.406 [INFO][90] felix/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/resources/v3/projectcalico.org/profiles" 2021-06-23 08:07:12.710 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:07:15.071 [INFO][90] felix/summary.go 100: Summarising 11 dataplane reconciliation loops over 1m3.1s: avg=5ms longest=20ms () 2021-06-23 08:07:42.557 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:07:42.711 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:08:06.739 [INFO][93] confd/watchercache.go 96: Watch channel closed by remote - recreate watcher ListRoot="/calico/resources/v3/projectcalico.org/bgppeers" 2021-06-23 08:08:12.712 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces 2021-06-23 08:08:18.442 [INFO][90] felix/summary.go 100: Summarising 8 dataplane reconciliation loops over 1m3.4s: avg=4ms longest=16ms () 2021-06-23 08:08:42.558 [INFO][91] monitor-addresses/startup.go 788: Using autodetected IPv4 address 10.10.16.82/24 on matching interface enahisic2i0 2021-06-23 08:08:42.713 [WARNING][90] felix/int_dataplane.go 878: Failed to auto-detect host MTU - no interfaces matched the MTU interface pattern. To use auto-MTU, set mtuIfacePattern to match your host's interfaces ^C root@ubuntu:~# kubectl logs -f calico-node-gnp9x -n kube-system
编辑kubectl edit ds calico-node -n kube-system
- name: IP_AUTODETECTION_METHOD value: "can-reach=10.10.16.254" #value: interface=en.*
成功了
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.81 | node specific | up | 08:25:14 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
Kubernetes-Calico百度云实践:集群外RR模式(微信)