开源集群管理软件 Load Balancer（Haproxy/Keepalived）

Direct Routing and the ARP Limitation

正常情况下，客户端发送请求，路由器会将目的主机的ip地址与mac地址关联，发达arp网络广播包，目的主机接到包后，将ip与mac地址写入arp缓存，保留15分钟，过期后重新更新。

Direct Routing负载均衡由于vip的存在，破坏ip与mac地址的对应关系的唯一性。导致arp请求出现vip关联多台主机，跳过路由直接被处理。

解决方法：
确保请求发送到路由器，而不是一个目标主机。通过过滤arp请求（arptables）或者过滤ip数据包（iptable firewalld）来实现。

除了上述3种方法外，还可以使用sysctl方式解决，即目标主机不通告arp请求的vip，不回复arp请求的vip。
# net.ipv4.conf.eth0.arp_ignore = 1
# net.ipv4.conf.eth0.arp_announce = 2

1. 静态解析
vm-4-14 & 15 & 16 & 17 执行

# cat >> /etc/hosts << EOF

192.168.4.14 vm-4-14
192.168.4.15 vm-4-15
192.168.4.16 vm-4-16
192.168.4.17 vm-4-17
EOF

2. ip转发
vm-4-14 & 15 执行

# cat >> /etc/sysctl.conf << EOF

net.ipv4.ip_forward = 1
net.ipv4.ip_nonlocal_bind = 1
EOF

# sysctl -p

3. arp问题
vm-4-16 & 17 执行

# cat >> /etc/sysctl.conf << EOF

net.ipv4.conf.eth0.arp_ignore = 1
net.ipv4.conf.eth0.arp_announce = 2
EOF

# sysctl -p

4. 部署keepalived
vm-4-14 & 15 执行

# yum install -y keepalived

vm-4-14 执行

# cat > /etc/keepalived/keepalived.conf << EOF
vrrp_script chk_haproxy {
  script "killall -0 haproxy" # check the haproxy process
  interval 2 # every 2 seconds
  weight 2 # add 2 points if OK
}

vrrp_instance VI_1 {
  interface eth0 # interface to monitor
  state MASTER # MASTER on haproxy, BACKUP on haproxy2
  virtual_router_id 51
  priority 101 # 101 on haproxy, 100 on haproxy2
  virtual_ipaddress {
     192.168.4.9/24 dev eth0 label eth0:0 # virtual ip address
  }
  track_script {
    chk_haproxy
  }
}
EOF

vm-4-15 执行

# cat > /etc/keepalived/keepalived.conf << EOF
vrrp_script chk_haproxy {
  script "killall -0 haproxy" # check the haproxy process
  interval 2 # every 2 seconds
  weight 2 # add 2 points if OK
}

vrrp_instance VI_1 {
  interface eth0 # interface to monitor
  state BACKUP # MASTER on haproxy, BACKUP on haproxy2
  virtual_router_id 51
  priority 100 # 101 on haproxy, 100 on haproxy2
  virtual_ipaddress {
     192.168.4.9/24 dev eth0 label eth0:0 # virtual ip address
  }
  track_script {
    chk_haproxy
  }
}
EOF

vm-4-14 & 15 执行

# systemctl start keepalived && systemctl enable keepalived

5. 证书生成
https://blog.csdn.net/weixin_40608446/article/details/104608255?utm_medium=distribute.pc_relevant.none-task-blog-2~default~baidujs_baidulandingword~default-0-104608255-blog-121972525.pc_relevant_aa2&spm=1001.2101.3001.4242.1&utm_relevant_index=3
https://www.cnblogs.com/bass6/p/6186971.html

# mkdir -p /etc/ssl/private
# openssl genrsa -out /etc/ssl/private/example.com.key 2048
# openssl req -new -key /etc/ssl/private/example.com.key -out /etc/ssl/private/example.com.csr
# openssl x509 -req -days 365 -in /etc/ssl/private/example.com.csr -signkey /etc/ssl/private/example.com.key -out /etc/ssl/private/example.com.crt
# cat /etc/ssl/private/example.com.crt /etc/ssl/private/example.com.key |tee /etc/ssl/private/example.com.pem

6. 部署haproxy
vm-4-14 & 15 执行

# yum install -y haproxy

# cat > /etc/haproxy/haproxy.cfg << EOF
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

listen stats
    mode http
    bind 0.0.0.0:9999
    stats enable
    log global
    stats uri /status
    stats auth admin:password

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend app_http *:80
    mode http
    default_backend server

frontend app_https
  bind 192.168.4.9:443 ssl crt /etc/ssl/private/example.com.pem
  default_backend server

backend server
    balance roundrobin
    mode http
    server  vm-4-16 192.168.4.16:80 check
    server  vm-4-17 192.168.4.17:80 check
EOF

# systemctl start haproxy && systemctl enable haproxy

7. 部署nginx
vm-4-16 & 17 执行

# yum install -y nginx

# systemctl start nginx && systemctl enable nginx

8. 测试

vm-4-14
# systemctl stop keepalived

vm-4-15
# systemctl stop haproxy

vm-4-17
# systemctl stop nginx

访问 http://192.168.4.9 正常