demo1
root@cloud:/etc/haproxy# cat haproxy.cfg #--------------------------------------------------------------------- # Example configuration for a possible web application. See the # full configuration options online. # # https://www.haproxy.org/download/2.1/doc/configuration.txt # https://cbonte.github.io/haproxy-dconv/2.1/configuration.html# #--------------------------------------------------------------------- #--------------------------------------------------------------------- # Global settings #--------------------------------------------------------------------- global # to have these messages end up in /var/log/haproxy.log you will # need to: # # 1) configure syslog to accept network log events. This is done # by adding the '-r' option to the SYSLOGD_OPTIONS in # /etc/sysconfig/syslog # # 2) configure local2 events to go to the /var/log/haproxy.log # file. A line like the following can be added to # /etc/sysconfig/syslog # # local2.* /var/log/haproxy.log # log 127.0.0.1 local2 # chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 4000 # user haproxy # group haproxy # daemon # turn on stats unix socket stats socket /var/lib/haproxy/stats #--------------------------------------------------------------------- # common defaults that all the 'listen' and 'backend' sections will # use if not designated in their block #--------------------------------------------------------------------- defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000 #--------------------------------------------------------------------- # main frontend which proxys to the backends #--------------------------------------------------------------------- frontend frr mode tcp bind *:9443 ## 监听9443端口 # bind *:443 ssl # To be completed .... acl url_static path_beg -i /static /images /javascript /stylesheets acl url_static path_end -i .jpg .gif .png .css .js default_backend frr #--------------------------------------------------------------------- # round robin balancing between the various backends #--------------------------------------------------------------------- backend frr mode tcp # 模式tcp balance roundrobin # 采用轮询的负载算法 server frr-10.10.18.34 10.10.18.34:80 check server frr-10.10.18.31 10.10.18.31:80 check
root@cloud:/etc/haproxy# conntrack -L -o ktimestamp | grep 9443 tcp 6 61 TIME_WAIT src=192.168.117.51 dst=10.10.16.47 sport=64741 dport=9443 src=10.10.16.47 dst=192.168.117.51 sport=9443 dport=64741 [ASSURED] mark=0 use=1 tcp 6 61 TIME_WAIT src=192.168.117.51 dst=10.10.16.47 sport=64740 dport=9443 src=10.10.16.47 dst=192.168.117.51 sport=9443 dport=64740 [ASSURED] mark=0 use=1 conntrack v1.4.4 (conntrack-tools): 114 flow entries have been shown. root@cloud:/etc/haproxy#
root@cloud:/etc/haproxy# tcpdump -i enahisic2i0 tcp and port 9443 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 17:54:07.919980 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 57, id 35264, offset 0, flags [DF], proto TCP (6), length 60) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [S], cksum 0xb694 (correct), seq 4212560958, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 290685940 ecr 0], length 0 17:54:07.920032 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.9443 > 192.168.117.51.64740: Flags [S.], cksum 0x5043 (incorrect -> 0xf96f), seq 1786083364, ack 4212560959, win 65160, options [mss 1460,sackOK,TS val 2010668810 ecr 290685940,nop,wscale 7], length 0 17:54:07.920179 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 57, id 35265, offset 0, flags [DF], proto TCP (6), length 60) 192.168.117.51.64741 > 10.10.16.47.9443: Flags [S], cksum 0x1cd0 (correct), seq 1574751036, win 64240, options [mss 1460,nop,wscale 8,sackOK,TS val 290685940 ecr 0], length 0 17:54:07.920203 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.9443 > 192.168.117.51.64741: Flags [S.], cksum 0x5043 (incorrect -> 0x2100), seq 1679736102, ack 1574751037, win 65160, options [mss 1460,sackOK,TS val 2010668810 ecr 290685940,nop,wscale 7], length 0 17:54:07.921034 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35266, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [.], cksum 0x22bf (correct), seq 1, ack 1, win 1029, options [nop,nop,TS val 290685941 ecr 2010668810], length 0 17:54:07.921220 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35268, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64741 > 10.10.16.47.9443: Flags [.], cksum 0x4a4f (correct), seq 1, ack 1, win 1029, options [nop,nop,TS val 290685941 ecr 2010668810], length 0 17:54:07.922589 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 607: (tos 0x0, ttl 57, id 35269, offset 0, flags [DF], proto TCP (6), length 593) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [P.], cksum 0xe890 (correct), seq 1:542, ack 1, win 1029, options [nop,nop,TS val 290685941 ecr 2010668810], length 541 17:54:07.922611 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 25550, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.9443 > 192.168.117.51.64740: Flags [.], cksum 0x503b (incorrect -> 0x22ab), seq 1, ack 542, win 505, options [nop,nop,TS val 2010668813 ecr 290685941], length 0 17:54:07.925985 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 246: (tos 0x0, ttl 64, id 25551, offset 0, flags [DF], proto TCP (6), length 232) 10.10.16.47.9443 > 192.168.117.51.64740: Flags [P.], cksum 0x50ef (incorrect -> 0x2fd6), seq 1:181, ack 542, win 505, options [nop,nop,TS val 2010668816 ecr 290685941], length 180 17:54:07.976725 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35273, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [.], cksum 0x1fb1 (correct), seq 542, ack 181, win 1028, options [nop,nop,TS val 290685997 ecr 2010668816], length 0 17:54:10.722333 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35275, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64741 > 10.10.16.47.9443: Flags [F.], cksum 0x3f5d (correct), seq 1, ack 1, win 1029, options [nop,nop,TS val 290688742 ecr 2010668810], length 0 17:54:10.722684 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35276, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [F.], cksum 0x14f6 (correct), seq 542, ack 181, win 1028, options [nop,nop,TS val 290688743 ecr 2010668816], length 0 17:54:10.723215 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 25552, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.9443 > 192.168.117.51.64740: Flags [F.], cksum 0x503b (incorrect -> 0x0c13), seq 181, ack 543, win 505, options [nop,nop,TS val 2010671613 ecr 290688743], length 0 17:54:10.723277 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 28597, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.9443 > 192.168.117.51.64741: Flags [.], cksum 0x503b (incorrect -> 0x3670), seq 1, ack 2, win 510, options [nop,nop,TS val 2010671614 ecr 290688742], length 0 17:54:10.723366 9c:52:f8:67:c4:d3 > f4:1d:6b:87:53:2a, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 64, id 28598, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.9443 > 192.168.117.51.64741: Flags [F.], cksum 0x503b (incorrect -> 0x366f), seq 1, ack 2, win 510, options [nop,nop,TS val 2010671614 ecr 290688742], length 0 17:54:10.725202 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35277, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64740 > 10.10.16.47.9443: Flags [.], cksum 0x0a06 (correct), seq 543, ack 182, win 1028, options [nop,nop,TS val 290688745 ecr 2010671613], length 0 17:54:10.725609 f4:1d:6b:87:53:2a > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 57, id 35278, offset 0, flags [DF], proto TCP (6), length 52) 192.168.117.51.64741 > 10.10.16.47.9443: Flags [.], cksum 0x3464 (correct), seq 2, ack 2, win 1029, options [nop,nop,TS val 290688746 ecr 2010671614], length 0 ^C 17 packets captured 17 packets received by filter 0 packets dropped by kernel
做了snat
[root@host-10-10-18-34 ~]# tcpdump -i eth0 tcp and port 80 -ennvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 17:51:24.896737 f4:1d:6b:87:53:2c > fa:16:3e:e3:68:97, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 4404, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.38682 > 10.10.18.34.80: Flags [S], cksum 0x2000 (correct), seq 2269574419, win 64240, options [mss 1460,sackOK,TS val 2482720490 ecr 0,nop,wscale 7], length 0 17:51:24.896814 fa:16:3e:e3:68:97 > f4:1d:6b:87:53:2c, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.18.34.80 > 10.10.16.47.38682: Flags [S.], cksum 0x3693 (incorrect -> 0x00a5), seq 1394405045, ack 2269574420, win 28960, options [mss 1460,sackOK,TS val 4063196440 ecr 2482720490,nop,wscale 7], length 0 17:51:24.897002 f4:1d:6b:87:53:2c > fa:16:3e:e3:68:97, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 63, id 4405, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.38682 > 10.10.18.34.80: Flags [R.], cksum 0x9e96 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 2482720491 ecr 4063196440], length 0 17:51:26.898560 f4:1d:6b:87:53:2c > fa:16:3e:e3:68:97, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 41452, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.38690 > 10.10.18.34.80: Flags [S], cksum 0xa666 (correct), seq 3304663328, win 64240, options [mss 1460,sackOK,TS val 2482722492 ecr 0,nop,wscale 7], length 0 17:51:26.898633 fa:16:3e:e3:68:97 > f4:1d:6b:87:53:2c, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.18.34.80 > 10.10.16.47.38690: Flags [S.], cksum 0x3693 (incorrect -> 0xd456), seq 977842796, ack 3304663329, win 28960, options [mss 1460,sackOK,TS val 4063198442 ecr 2482722492,nop,wscale 7], length 0 17:51:26.898810 f4:1d:6b:87:53:2c > fa:16:3e:e3:68:97, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 63, id 41453, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.38690 > 10.10.18.34.80: Flags [R.], cksum 0x7249 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 2482722492 ecr 4063198442], length 0 ^C 6 packets captured 6 packets received by filter 0 packets dropped by kernel [root@host-10-10-18-34 ~]#
[root@host-10-10-18-31 ~]# tcpdump -i eth0 tcp and port 80 -ennvv tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes 17:51:01.527070 f4:1d:6b:87:53:2c > fa:16:3e:97:87:9e, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 22721, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.39020 > 10.10.18.31.80: Flags [S], cksum 0x0425 (correct), seq 2516272890, win 64240, options [mss 1460,sackOK,TS val 235441870 ecr 0,nop,wscale 7], length 0 17:51:01.527619 fa:16:3e:97:87:9e > f4:1d:6b:87:53:2c, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.18.31.80 > 10.10.16.47.39020: Flags [S.], cksum 0x3690 (incorrect -> 0x8c3e), seq 856947989, ack 2516272891, win 28960, options [mss 1460,sackOK,TS val 3305442423 ecr 235441870,nop,wscale 7], length 0 17:51:01.529069 f4:1d:6b:87:53:2c > fa:16:3e:97:87:9e, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 63, id 22722, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.39020 > 10.10.18.31.80: Flags [R.], cksum 0x2a2f (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 235441872 ecr 3305442423], length 0 17:51:03.528344 f4:1d:6b:87:53:2c > fa:16:3e:97:87:9e, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 63, id 38753, offset 0, flags [DF], proto TCP (6), length 60) 10.10.16.47.39028 > 10.10.18.31.80: Flags [S], cksum 0xc2f6 (correct), seq 3174652176, win 64240, options [mss 1460,sackOK,TS val 235443872 ecr 0,nop,wscale 7], length 0 17:51:03.528389 fa:16:3e:97:87:9e > f4:1d:6b:87:53:2c, ethertype IPv4 (0x0800), length 74: (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) 10.10.18.31.80 > 10.10.16.47.39028: Flags [S.], cksum 0x3690 (incorrect -> 0xa5c3), seq 2938969719, ack 3174652177, win 28960, options [mss 1460,sackOK,TS val 3305444424 ecr 235443872,nop,wscale 7], length 0 17:51:03.528561 f4:1d:6b:87:53:2c > fa:16:3e:97:87:9e, ethertype IPv4 (0x0800), length 66: (tos 0x0, ttl 63, id 38754, offset 0, flags [DF], proto TCP (6), length 52) 10.10.16.47.39028 > 10.10.18.31.80: Flags [R.], cksum 0x43b5 (correct), seq 1, ack 1, win 502, options [nop,nop,TS val 235443873 ecr 3305444424], length 0 ^C 6 packets captured 6 packets received by filter 0 packets dropped by kernel
demo2: haproxy + keepalived
节点1
节点2
节点3
keepalived
[root@centos7 keepalived]# cat keepalived.conf global_defs { script_user root enable_script_security } vrrp_script chk_haproxy { script "/bin/bash -c 'if [[ $(netstat -nlp | grep 9443) ]]; then exit 0; else exit 1; fi'" # haproxy 检测 interval 2 # 每2秒执行一次检测 weight 11 # 权重变化 } vrrp_instance VI_1 { interface enp125s0f0 ###宿主机网卡名 state BACKUP virtual_router_id 51 # id设为相同，表示是同一个虚拟路由组 priority 100 #初始权重 nopreempt #不抢占 unicast_peer { } virtual_ipaddress { 10.10.16.249 # vip } authentication { auth_type PASS auth_pass password } track_script { chk_haproxy } notify "/container/service/keepalived/assets/" }
vip
[root@centos7 ~]# ip a | grep 10.10.16.249
inet 10.10.16.249/32 scope global enp125s0f0
[root@centos7 ~]#
[root@centos7 ~]# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enp125s0f0 [root@centos7 ~]# systemctl status haproxy ● haproxy.service - HAProxy Load Balancer Loaded: loaded (/usr/lib/systemd/system/haproxy.service; disabled; vendor preset: disabled) Active: active (running) since Sun 2021-06-27 22:32:35 EDT; 24min ago Main PID: 129054 (haproxy-systemd) CGroup: /system.slice/haproxy.service ├─129054 /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid ├─129055 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds └─129056 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds Jun 27 22:32:35 centos7 systemd[1]: Started HAProxy Load Balancer. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: haproxy-systemd-wrapper: executing /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : parsing [/etc/haproxy/haproxy.cfg:41] : 'option httplog' not usable with frontend 'frr' (ne...on tcplog'. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : config : 'option forwardfor' ignored for frontend 'frr' as it requires HTTP mode. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : config : 'option forwardfor' ignored for backend 'frr' as it requires HTTP mode. Hint: Some lines were ellipsized, use -l to show in full. [root@centos7 ~]#
stop centos7上的haproxy
[root@centos7 ~]# systemctl stop haproxy [root@centos7 ~]# systemctl status haproxy ● haproxy.service - HAProxy Load Balancer Loaded: loaded (/usr/lib/systemd/system/haproxy.service; disabled; vendor preset: disabled) Active: inactive (dead) Jun 27 22:32:35 centos7 systemd[1]: Started HAProxy Load Balancer. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: haproxy-systemd-wrapper: executing /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : parsing [/etc/haproxy/haproxy.cfg:41] : 'option httplog' not usable with frontend 'frr' (ne...on tcplog'. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : config : 'option forwardfor' ignored for frontend 'frr' as it requires HTTP mode. Jun 27 22:32:35 centos7 haproxy-systemd-wrapper[129054]: [WARNING] 177/223235 (129055) : config : 'option forwardfor' ignored for backend 'frr' as it requires HTTP mode. Jun 27 22:57:19 centos7 systemd[1]: Stopping HAProxy Load Balancer... Jun 27 22:57:19 centos7 haproxy-systemd-wrapper[129054]: haproxy-systemd-wrapper: SIGTERM -> 129056. Jun 27 22:57:19 centos7 haproxy-systemd-wrapper[129054]: haproxy-systemd-wrapper: exit, haproxy RC=0 Jun 27 22:57:19 centos7 systemd[1]: Stopped HAProxy Load Balancer. Hint: Some lines were ellipsized, use -l to show in full. [root@centos7 ~]#
ip 沒有切換到其他
[root@centos7 ~]# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enp125s0f0 [root@centos7 ~]#
root@cloud:/etc/keepalived# tcpdump -i enahisic2i0 vrrp -nnvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 11:17:51.463086 IP (tos 0xc0, ttl 255, id 2410, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 224.0.0.18: vrrp 10.10.16.251 > 224.0.0.18: VRRPv2, Advertisement, vrid 51, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:17:51.463545 IP (tos 0xc0, ttl 255, id 10954, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.5 > 224.0.0.18: vrrp 10.10.16.5 > 224.0.0.18: VRRPv2, Advertisement, vrid 125, prio 1, authtype simple, intvl 1s, length 20, addrs: 10.10.16.4 auth "OCqTw1kP" 11:17:51.464665 IP (tos 0xc0, ttl 255, id 46168, offset 0, flags [none], proto VRRP (112), length 40) 10.17.17.11 > 224.0.0.18: vrrp 10.17.17.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 201, prio 2, authtype simple, intvl 1s, length 20, addrs: 10.17.17.201 auth "c6pldPc0" 11:17:51.465799 IP (tos 0xc0, ttl 255, id 13601, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.32 > 224.0.0.18: vrrp 10.10.16.32 > 224.0.0.18: VRRPv2, Advertisement, vrid 227, prio 100, authtype none, intvl 1s, length 20, addrs: 192.168.1.100 11:17:52.463462 IP (tos 0xc0, ttl 255, id 2411, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 224.0.0.18: vrrp 10.10.16.251 > 224.0.0.18: VRRPv2, Advertisement, vrid 51, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:17:52.464741 IP (tos 0xc0, ttl 255, id 10955, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.5 > 224.0.0.18: vrrp 10.10.16.5 > 224.0.0.18: VRRPv2, Advertisement, vrid 125, prio 1, authtype simple, intvl 1s, length 20, addrs: 10.10.16.4 auth "OCqTw1kP" 11:17:52.464767 IP (tos 0xc0, ttl 255, id 46169, offset 0, flags [none], proto VRRP (112), length 40) 10.17.17.11 > 224.0.0.18: vrrp 10.17.17.11 > 224.0.0.18: VRRPv2, Advertisement, vrid 201, prio 2, authtype simple, intvl 1s, length 20, addrs: 10.17.17.201 auth "c6pldPc0" 11:17:52.465916 IP (tos 0xc0, ttl 255, id 13602, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.32 > 224.0.0.18: vrrp 10.10.16.32 > 224.0.0.18: VRRPv2, Advertisement, vrid 227, prio 100, authtype none, intvl 1s, length 20, addrs: 192.168.1.100 ^C
原來是vrid 51 被占用了，更改vrrid
root@cloud:/etc/keepalived# vi keepalived.conf global_defs { script_user root enable_script_security } vrrp_script chk_haproxy { script "/bin/bash -c 'if [[ $(netstat -nlp | grep 9443) ]]; then exit 0; else exit 1; fi'" # haproxy 检测 interval 2 # 每2秒执行一次检测 weight 11 # 权重变化 } vrrp_instance VI_1 { interface enahisic2i0 ###宿主机网卡名 state BACKUP virtual_router_id 61 # id设为相同，表示是同一个虚拟路由组 priority 100 #初始权重 nopreempt #不抢占 unicast_peer { } virtual_ipaddress { 10.10.16.249 # vip } authentication { auth_type PASS auth_pass password } track_script { chk_haproxy } notify "/container/service/keepalived/assets/" }
root@cloud:/etc/keepalived# tcpdump -i enahisic2i0 vrrp and host 10.10.16.47 -nnvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 11:27:48.887134 IP (tos 0xc0, ttl 255, id 41, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.47 > 224.0.0.18: vrrp 10.10.16.47 > 224.0.0.18: VRRPv2, Advertisement, vrid 61, prio 111, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:27:49.887294 IP (tos 0xc0, ttl 255, id 42, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.47 > 224.0.0.18: vrrp 10.10.16.47 > 224.0.0.18: VRRPv2, Advertisement, vrid 61, prio 111, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:27:50.887452 IP (tos 0xc0, ttl 255, id 43, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.47 > 224.0.0.18: vrrp 10.10.16.47 > 224.0.0.18: VRRPv2, Advertisement, vrid 61, prio 111, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:27:51.887606 IP (tos 0xc0, ttl 255, id 44, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.47 > 224.0.0.18: vrrp 10.10.16.47 > 224.0.0.18: VRRPv2, Advertisement, vrid 61, prio 111, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 11:27:52.887746 IP (tos 0xc0, ttl 255, id 45, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.47 > 224.0.0.18: vrrp 10.10.16.47 > 224.0.0.18: VRRPv2, Advertisement, vrid 61, prio 111, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" ^C 5 packets captured 6 packets received by filter 0 packets dropped by kernel root@cloud:/etc/keepalived#
root@cloud:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@cloud:/etc/keepalived#
root@cloud:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@cloud:/etc/keepalived# systemctl stop haproxy root@cloud:/etc/keepalived# systemctl stop haproxy root@cloud:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@cloud:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@cloud:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@cloud:/etc/keepalived#
root@ubuntu:/etc/keepalived# ip a sh enahisic2i0 2: enahisic2i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 48:57:02:64:e7:ab brd ff:ff:ff:ff:ff:ff inet 10.10.16.82/24 brd 10.10.16.255 scope global enahisic2i0 valid_lft forever preferred_lft forever inet 10.10.16.249/32 scope global enahisic2i0 valid_lft forever preferred_lft forever inet6 fe80::4a57:2ff:fe64:e7ab/64 scope link valid_lft forever preferred_lft forever root@ubuntu:/etc/keepalived#
抓不到vrrp报文
root@ubuntu:/etc/keepalived# tcpdump -i enahisic2i0 vrrp and host 10.10.16.82 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes
抓不到vrrp报文
[root@centos7 keepalived]# tcpdump -i enp125s0f0 vrrp and host 10.10.16.251 -n tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes ^C 0 packets captured 0 packets received by filter 0 packets dropped by kernel [root@centos7 keepalived]#
原来是单播地址为空，
unicast_peer {
}
给各个节点添加单播地址
unicast_peer { 10.10.16.47 10.10.16.251 }
unicast_peer { 10.10.16.47 10.10.16.251 }
unicast_peer { 10.10.16.251 10.10.16.82 }
root@ubuntu:/etc/keepalived# tcpdump -i enahisic2i0 vrrp and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 14:11:36.171506 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 60: (tos 0xc0, ttl 255, id 14966, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.82: vrrp 10.10.16.251 > 10.10.16.82: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:11:37.172029 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 60: (tos 0xc0, ttl 255, id 14968, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.82: vrrp 10.10.16.251 > 10.10.16.82: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:11:38.172558 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 60: (tos 0xc0, ttl 255, id 14970, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.82: vrrp 10.10.16.251 > 10.10.16.82: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:11:39.173066 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 60: (tos 0xc0, ttl 255, id 14972, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.82: vrrp 10.10.16.251 > 10.10.16.82: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:11:40.173561 b0:08:75:5f:b8:5b > 48:57:02:64:e7:ab, ethertype IPv4 (0x0800), length 60: (tos 0xc0, ttl 255, id 14974, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.82: vrrp 10.10.16.251 > 10.10.16.82: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" ^C 5 packets captured 5 packets received by filter 0 packets dropped by kernel root@ubuntu:/etc/keepalived#
root@cloud:/etc/keepalived# tcpdump -i enahisic2i0 vrrp and host 10.10.16.47 -nnvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 14:12:05.187719 IP (tos 0xc0, ttl 255, id 15023, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:06.188234 IP (tos 0xc0, ttl 255, id 15025, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:07.188775 IP (tos 0xc0, ttl 255, id 15027, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:08.189287 IP (tos 0xc0, ttl 255, id 15029, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:09.189825 IP (tos 0xc0, ttl 255, id 15031, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:10.190313 IP (tos 0xc0, ttl 255, id 15033, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:11.190869 IP (tos 0xc0, ttl 255, id 15035, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:12.191405 IP (tos 0xc0, ttl 255, id 15037, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:13.191911 IP (tos 0xc0, ttl 255, id 15039, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:14.192456 IP (tos 0xc0, ttl 255, id 15041, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:12:15.192967 IP (tos 0xc0, ttl 255, id 15043, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.251 > 10.10.16.47: vrrp 10.10.16.251 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" ^C 11 packets captured 11 packets received by filter 0 packets dropped by kernel root@cloud:/etc/keepalived#
[root@centos7 keepalived]# tcpdump -i enp125s0f0 vrrp and host 10.10.16.251 -n tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on enp125s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes 02:12:56.708029 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:12:57.708177 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:12:58.708304 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:12:59.708701 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:13:00.709329 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:13:01.709717 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:13:02.710328 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:13:03.710734 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 02:13:04.711359 IP 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 121, authtype simple, intvl 1s, length 20 ^C 9 packets captured 11 packets received by filter 0 packets dropped by kernel [root@centos7 keepalived]#
各个节点的haproxy服务正常
停止ubuntu上的haproxy
root@ubuntu:/etc/keepalived# ip a | grep 10.10.16.249 inet 10.10.16.249/32 scope global enahisic2i0 root@ubuntu:/etc/keepalived#
root@ubuntu:/etc/keepalived# tcpdump -i enahisic2i0 vrrp and host 10.10.16.82 -ennvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 14:18:15.876583 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 649, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.47: vrrp 10.10.16.82 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:15.876615 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 650, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.251: vrrp 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:16.877292 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 651, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.47: vrrp 10.10.16.82 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:16.877317 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 652, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.251: vrrp 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:17.877792 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 653, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.47: vrrp 10.10.16.82 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:17.877822 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 654, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.251: vrrp 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:18.878490 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 655, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.47: vrrp 10.10.16.82 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:18.878515 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 656, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.251: vrrp 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:19.878962 48:57:02:64:e7:ab > 9c:52:f8:67:c4:d3, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 657, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.47: vrrp 10.10.16.82 > 10.10.16.47: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" 14:18:19.878992 48:57:02:64:e7:ab > b0:08:75:5f:b8:5b, ethertype IPv4 (0x0800), length 54: (tos 0xc0, ttl 255, id 658, offset 0, flags [none], proto VRRP (112), length 40) 10.10.16.82 > 10.10.16.251: vrrp 10.10.16.82 > 10.10.16.251: VRRPv2, Advertisement, vrid 61, prio 100, authtype simple, intvl 1s, length 20, addrs: 10.10.16.249 auth "password" ^C 10 packets captured 10 packets received by filter 0 packets dropped by kernel
keepalived 选举
keepalived，1个要点，priority 100 #初始权重，BACKUP 节点的一定要比这个要小；
1.“weight”值为正数时
在vrrp_script中指定的脚本如果检测成功，那么Master节点的权值将是“weight值与”priority“值之和，如果脚本检测失败，那么Master节点的权值保持为“priority”值，因此切换策略为：
Master节点“vrrp_script”脚本检测失败时，如果Master节点“priority”值小于Backup节点“weight值与”priority“值之和，将发生主、备切换。
Master节点“vrrp_script”脚本检测成功时，如果Master节点“weight”值与“priority”值之和大于Backup节点“weight”值与“priority”值之和，主节点依然为主节点，不发生切换。
2.“weight”值为负数时
在“vrrp_script”中指定的脚本如果检测成功，那么Master节点的权值仍为“priority”值，当脚本检测失败时，Master节点的权值将是“priority“值与“weight”值之差，因此切换策略为：
节点“vrrp_script”脚本检测失败时，如果Master节点“priority”值与“weight”值之差小于Backup节点“priority”值，将发生主、备切换。
Master节点“vrrp_script”脚本检测成功时，如果Master节点“priority”值大于Backup节点“priority”值时，主节点依然为主节点，不发生切换。
在熟悉了Keepalived主、备角色的选举策略后，再来分析一下刚才实例，由于A、B两个节点设置的“weight”值都为10，因此符合选举策略的第一种，在A节点停止Mysql服务后，A节点的脚本检测将失败，此时A节点的权值将保持为A节点上设置的“priority”值，即为100，而B节点的权值将变为“weight”值与“priority”值之和，也就是90（10+80），这样就出现了A节点权值仍然大于B节点权值的情况，因此不会发生主、备切换。
对于“weight”值的设置，有一个简单的标准，即“weight”值的绝对值要大于Master和Backup节点“priority”值之差。对于上面A、B两个节点的例子，只要设置“weight”值大于20即可保证集群正常运行和切换。由此可见，对于“weight值的设置，要非常谨慎，如果设置不好，将导致集群角色选举失败，使集群陷于瘫痪状态。