0. 组件版本
docker-1.13.1
calico-v1.0.2
1. 安装calico(已安装etcd、docker)
a.创建/etc/calico/calicoctl.cfg
apiVersion: v1
kind: calicoApiConfig
metadata:
spec:
datastoreType: "etcdv2"
etcdEndpoints: http://192.168.56.201:2379,http://192.168.56.202:2379
b.启动calico
calicoctl node run
c.更新ipPool
calicoctl get ipPool -o yaml > ipPool.yaml
calicoctl apply -f ipPool.yaml
d.更新docker配置文件 /etc/docker/daemon.json (并重启docker服务)
"cluster-store": "etcd://192.168.56.201:2379"
e. 创建docker网络
docker network create --driver calico --ipam-driver calico-ipam [--subnet=192.168.100.0/24] cal_net1
#calico默认基于ipPool(192.168.0.0/16)创建26位掩码的子网
f. 创建docker实例，sample如下：
docker run --net=cal_net1 -e MYSQL_ROOT_PASSWORD=123456 --name mysql-01 -tid mysql:5.6
小结：
1.可创建多个network, 默认情况下同节点不同network的多个实例属于同一子网
2.不同network之间相互隔离(iptables)
2. Route table
instance:
root@000fb102dec1:/# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
6: cali0@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff
inet 192.168.10.128/32 scope global cali0
valid_lft forever preferred_lft forever
inet6 fe80::ecee:eeff:feee:eeee/64 scope link
valid_lft forever preferred_lft forever
root@000fb102dec1:/# ip route show
default via 169.254.1.1 dev cali0
169.254.1.1 dev cali0 scope link
host:
[root@k01 calico]# ip route show
default via 10.0.2.2 dev enp0s3
10.0.2.0/24 dev enp0s3 proto kernel scope link src 10.0.2.15
192.168.10.128 dev calife6406257b4 scope link
blackhole 192.168.10.128/26 proto bird
192.168.10.192/26 via 192.168.100.202 dev enp0s9 proto bird
192.168.100.0/24 dev enp0s9 proto kernel scope link src 192.168.100.201
小结:
1.docker实例mac为ee:ee:ee:ee:ee:ee, 默认路由为169.254.1.1(由于宿主机内veth已开启arp_proxy,/proc/sys/net/ipv4/conf/calife6406257b4/proxy_arp)
2.默认有route到其它节点的表项,如(192.168.10.192/26 via 192.168.100.202 dev enp0s9 proto bird)
3. Policy
a.network默认的policy
calicoctl get profile cal_net2 -o yaml > cal_net2_policy.yaml
- apiVersion: v1
kind: profile
metadata:
name: cal_net2
tags:
- cal_net2
spec:
egress:
- action: allow
destination: {}
source: {}
ingress:
- action: allow
destination: {}
source:
tag: cal_net2
- action: allow
protocol: icmp
source:
tag: cal_net1
b.允许cal_net1实例ping cal_net2
实例如下：cal_net2_policy.yaml
- apiVersion: v1
kind: profile
metadata:
name: cal_net2
tags:
- cal_net2
spec:
egress:
- action: allow
destination: {}
source: {}
ingress:
- action: allow
destination: {}
source:
tag: cal_net2
- action: allow
protocol: icmp
source:
tag: cal_net1
c. calicoctl apply -f cal_net2_policy.yaml
d.policy的实现：
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
num pkts bytes target prot opt in out source destination
1 1532 148K felix-INPUT all -- * * 0.0.0.0/0 0.0.0.0/0
2 18274 2610K KUBE-FIREWALL all -- * * 0.0.0.0/0 0.0.0.0/0
Chain FORWARD (policy DROP 0 packets, 0 bytes)
num pkts bytes target prot opt in out source destination
1 0 0 felix-FORWARD all -- * * 0.0.0.0/0 0.0.0.0/0
2 0 0 DOCKER-ISOLATION all -- * * 0.0.0.0/0 0.0.0.0/0
3 0 0 DOCKER all -- * docker0 0.0.0.0/0 0.0.0.0/0
4 0 0 ACCEPT all -- * docker0 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
5 0 0 ACCEPT all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
6 0 0 ACCEPT all -- docker0 docker0 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 2 packets, 120 bytes)
num pkts bytes target prot opt in out source destination
1 1462 153K felix-OUTPUT all -- * * 0.0.0.0/0 0.0.0.0/0
2 20663 2350K KUBE-FIREWALL all -- * * 0.0.0.0/0 0.0.0.0/0
3 21051 2394K KUBE-SERVICES all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */
Chain DOCKER (1 references)
num pkts bytes target prot opt in out source destination
Chain DOCKER-ISOLATION (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain KUBE-FIREWALL (2 references)
num pkts bytes target prot opt in out source destination
1 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
Chain KUBE-SERVICES (1 references)
num pkts bytes target prot opt in out source destination
Chain felix-FAILSAFE-IN (0 references)
num pkts bytes target prot opt in out source destination
1 0 0 ACCEPT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:22
Chain felix-FAILSAFE-OUT (0 references)
num pkts bytes target prot opt in out source destination
1 0 0 ACCEPT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:2379
2 0 0 ACCEPT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:2380
3 0 0 ACCEPT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:4001
4 0 0 ACCEPT tcp -- * * 0.0.0.0/0 0.0.0.0/0 tcp dpt:7001
Chain felix-FORWARD (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate INVALID
2 0 0 ACCEPT all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
3 0 0 felix-FROM-ENDPOINT all -- cali+ * 0.0.0.0/0 0.0.0.0/0
4 0 0 felix-TO-ENDPOINT all -- * cali+ 0.0.0.0/0 0.0.0.0/0
5 0 0 ACCEPT all -- cali+ * 0.0.0.0/0 0.0.0.0/0
6 0 0 ACCEPT all -- * cali+ 0.0.0.0/0 0.0.0.0/0
7 0 0 felix-FROM-HOST-IF all -- * * 0.0.0.0/0 0.0.0.0/0
8 0 0 felix-TO-HOST-IF all -- * * 0.0.0.0/0 0.0.0.0/0
Chain felix-FROM-ENDPOINT (2 references)
num pkts bytes target prot opt in out source destination
1 0 0 felix-from-1340c926440 all -- cali1340c926440 * 0.0.0.0/0 0.0.0.0/0 [goto]
2 0 0 felix-from-7fd158d9982 all -- cali7fd158d9982 * 0.0.0.0/0 0.0.0.0/0 [goto]
3 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* From unknown endpoint */
Chain felix-FROM-HOST-IF (2 references)
num pkts bytes target prot opt in out source destination
1 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 /* Unknown interface, return */
Chain felix-INPUT (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate INVALID
2 1534 149K ACCEPT all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
3 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfbffffff
4 0 0 MARK all -- cali+ * 0.0.0.0/0 0.0.0.0/0 MARK or 0x4000000
5 0 0 felix-FROM-HOST-IF all -- * * 0.0.0.0/0 0.0.0.0/0 [goto] mark match 0x0/0x4000000
6 0 0 ACCEPT udp -- * * 0.0.0.0/0 0.0.0.0/0 udp spt:68 dpt:67
7 0 0 ACCEPT udp -- * * 0.0.0.0/0 0.0.0.0/0 udp dpt:53
8 0 0 felix-FROM-ENDPOINT all -- * * 0.0.0.0/0 0.0.0.0/0
Chain felix-OUTPUT (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate INVALID
2 1454 152K ACCEPT all -- * * 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
3 8 480 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfbffffff
4 0 0 MARK all -- * cali+ 0.0.0.0/0 0.0.0.0/0 MARK or 0x4000000
5 8 480 felix-TO-HOST-IF all -- * * 0.0.0.0/0 0.0.0.0/0 [goto] mark match 0x0/0x4000000
Chain felix-TO-ENDPOINT (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 felix-to-1340c926440 all -- * cali1340c926440 0.0.0.0/0 0.0.0.0/0 [goto]
2 0 0 felix-to-7fd158d9982 all -- * cali7fd158d9982 0.0.0.0/0 0.0.0.0/0 [goto]
3 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* To unknown endpoint */
Chain felix-TO-HOST-IF (2 references)
num pkts bytes target prot opt in out source destination
1 8 480 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 /* Unknown interface, return */
Chain felix-from-1340c926440 (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfeffffff
2 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 MAC ! EE:EE:EE:EE:EE:EE /* Incorrect source MAC */
3 0 0 felix-p-cal_net2-o all -- * * 0.0.0.0/0 0.0.0.0/0
4 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000 /* Profile accepted packet */
5 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* Packet did not match any profile (endpoint 1340c926440796cd5b2ff7a921ace1499e84f9ae9ec227f5176e34045efc4d7d) */
Chain felix-from-7fd158d9982 (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfeffffff
2 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 MAC ! EE:EE:EE:EE:EE:EE /* Incorrect source MAC */
3 0 0 felix-p-cal_net1-o all -- * * 0.0.0.0/0 0.0.0.0/0
4 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000 /* Profile accepted packet */
5 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* Packet did not match any profile (endpoint 7fd158d998288a130703e07f3518e95ce2826cb47d4225f27becef19538e5242) */
Chain felix-p-cal_net1-i (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 match-set felix-4-t:5pLLaySaRFlvkik7lQrHp src MARK or 0x1000000
2 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000
Chain felix-p-cal_net1-o (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK or 0x1000000
2 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000
Chain felix-p-cal_net2-i (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 match-set felix-4-t:6Z4IITNFLy-q3zhZ8hcIU src MARK or 0x1000000
2 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000
3 0 0 MARK icmp -- * * 0.0.0.0/0 0.0.0.0/0 match-set felix-4-t:5pLLaySaRFlvkik7lQrHp src MARK or 0x1000000
4 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000
Chain felix-p-cal_net2-o (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK or 0x1000000
2 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000
Chain felix-to-1340c926440 (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfeffffff
2 0 0 felix-p-cal_net2-i all -- * * 0.0.0.0/0 0.0.0.0/0
3 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000 /* Profile accepted packet */
4 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* Packet did not match any profile (endpoint 1340c926440796cd5b2ff7a921ace1499e84f9ae9ec227f5176e34045efc4d7d) */
Chain felix-to-7fd158d9982 (1 references)
num pkts bytes target prot opt in out source destination
1 0 0 MARK all -- * * 0.0.0.0/0 0.0.0.0/0 MARK and 0xfeffffff
2 0 0 felix-p-cal_net1-i all -- * * 0.0.0.0/0 0.0.0.0/0
3 0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0 mark match 0x1000000/0x1000000 /* Profile accepted packet */
4 0 0 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 /* Packet did not match any profile (endpoint 7fd158d998288a130703e07f3518e95ce2826cb47d4225f27becef19538e5242) */
参考文档：
https://www.cnblogs.com/netonline/p/9720279.html
https://www.cnblogs.com/CloudMan6/p/7509975.html
https://blog.csdn.net/networken/article/details/84314094
https://docs.projectcalico.org/v3.5/reference/calicoctl/resources/globalnetworkpolicy
https://www.cnblogs.com/laiyuan/p/8607755.html