容器化haproxy+keepalived

# 拉取haproxy镜像
docker pull haproxy:1.7.8-alpine
mkdir /etc/haproxy
cat >/etc/haproxy/haproxy.cfg<<EOF
global
  log 127.0.0.1 local0 err
  maxconn 50000
  uid 99
  gid 99
  #daemon
  nbproc 1
  pidfile haproxy.pid

defaults
  mode http
  log 127.0.0.1 local0 err
  maxconn 50000
  retries 3
  timeout connect 5s
  timeout client 30s
  timeout server 30s
  timeout check 2s

listen admin_stats
  mode http
  bind 0.0.0.0:1080
  log 127.0.0.1 local0 err
  stats refresh 30s
  stats uri     /haproxy-status
  stats realm   Haproxy Statistics
  stats auth    will:will
  stats hide-version
  stats admin if TRUE

frontend k8s-https
  bind 0.0.0.0:8443
  mode tcp
  #maxconn 50000
  default_backend k8s-https

backend k8s-https
  mode tcp
  balance roundrobin
  server lab1 192.168.105.92:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
  server lab2 192.168.105.93:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
  server lab3 192.168.105.94:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
EOF

# 启动haproxy
docker run -d --name my-haproxy 
-v /etc/haproxy:/usr/local/etc/haproxy:ro 
-p 8443:8443 
-p 1080:1080 
--restart always 
haproxy:1.7.8-alpine

# 查看日志
docker logs my-haproxy

# 浏览器查看状态
http://192.168.105.92:1080/haproxy-status
http://192.168.105.93:1080/haproxy-status
http://192.168.105.94:1080/haproxy-status

# 拉取keepalived镜像
docker pull osixia/keepalived:1.4.4

# 启动
# 载入内核相关模块
lsmod | grep ip_vs
modprobe ip_vs

# 启动keepalived
# ens32为本次实验192.168.105.0/24网段的所在网卡
docker run --net=host --cap-add=NET_ADMIN 
-e KEEPALIVED_INTERFACE=ens32 
-e KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['192.168.105.99']" 
-e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.105.92','192.168.105.93','192.168.105.94']" 
-e KEEPALIVED_PASSWORD=hello 
--name k8s-keepalived 
--restart always 
-d osixia/keepalived:1.4.4

# 查看日志
# 会看到两个成为backup 一个成为master
docker logs k8s-keepalived

# 此时会配置 192.168.105.99 到其中一台机器
# ping测试
ping -c4 192.168.105.99

# 如果失败后清理后，重新实验
#docker rm -f k8s-keepalived
#ip a del 192.168.105.99/32 dev ens32

测试Haproxy高可用

#抓个包看看现在VIP在哪台机器上，然后去关掉这台机器

tcpdump -nn host 224.0.100.100

[root@master2 ~]# tcpdump -nn host 224.0.100.100
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:45:59.768033 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:46:00.769503 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:46:01.771062 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
^C

#现在VIP是在192.168.1.4这台主机上，关掉这台机器
init 0

#关掉之后发现192.168.1.5立马接管了。现在VIP是在1.5上
[root@node3 ~]# tcpdump -nn host 224.0.100.100
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes
17:48:25.031679 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:26.033805 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:27.035313 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:28.036628 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:29.039011 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:30.041249 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:31.043065 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:32.045007 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:33.046781 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:34.048776 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:35.051280 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 100, authtype simple, intvl 1s, length 20
17:48:35.929482 IP 192.168.1.4 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 0, authtype simple, intvl 1s, length 20
17:48:36.618749 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:37.699849 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:38.700669 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:39.702840 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:40.704254 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:41.706221 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
17:48:42.706478 IP 192.168.1.5 > 224.0.100.100: VRRPv2, Advertisement, vrid 6, prio 80, authtype simple, intvl 1s, length 20
^C
19 packets captured
326 packets received by filter
172 packets dropped by kernel

#切换到master2，再创建个POD试试
kubectl run curl --image=radial/busyboxplus:curl -i --tty
date
exit
kubectl delete deployment curl

[root@master2 ~]# kubectl run curl --image=radial/busyboxplus:curl -i --tty
If you don't see a command prompt, try pressing enter.
[ root@curl-87b54756-xfgrn:/ ]$ date
Thu Sep 13 09:50:58 UTC 2018
[ root@curl-87b54756-xfgrn:/ ]$
[ root@curl-87b54756-xfgrn:/ ]$ exit
Session ended, resume using 'kubectl attach curl-87b54756-xfgrn -c curl -i -t' command when the pod is running
[root@master2 ~]# kubectl delete deployment curl
deployment.extensions "curl" deleted