LVS配置 NAT和DR模式配置负载均衡

LVS配置

搭建NAT模式的HTTP负载集群

本次环境为：

环境	DIP	VIP	需要安装的应用	系统版本
client	192.168.100.2	/	/	RedHat 8
DR	192.168.100.3	192.168.222.250	ipvsadm	RedHat 8
RS1	192.168.100.4	/	httpd	RedHat 8
RS2	192.168.100.5	/	httpd	RedHat 8

准备工作：

DR需要配置2块网卡

#DR：
[root@DR ~]# systemctl disable --now firewalld
[root@DR ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@DR ~]# setenforce 0
[root@DR ~]# yum -y install ipvsadm

#RS1：
[root@RS1 ~]# systemctl disable --now firewalld
[root@RS1 ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@RS1 ~]# setenforce 0
[root@RS1 ~]# yum -y install httpd net-tools
[root@RS1 ~]# systemctl enable --now httpd
[root@RS1 ~]# echo lvs-web1 > /var/www/html/index.html
#将网关指向DIP
[root@RS1 ~]# route add default gw 192.168.100.3

#RS2：
[root@RS2 ~]# systemctl disable --now firewalld
[root@RS2 ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@RS2 ~]# setenforce 0
[root@RS2 ~]# yum -y install httpd net-tools
[root@RS2 ~]# systemctl enable --now httpd
[root@RS2 ~]# echo lvs-web2 > /var/www/html/index.html
#将网关指向DIP
[root@RS2 ~]# route add default gw 192.168.100.3

开启IP转发

配置NAT模式下的ip转发，让通过Load Balancer的ip包能够转发到真正提供服务的Real Server之上进行处理

#DR
[root@DR ~]# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1

[root@DR ~]# sysctl -p
net.ipv4.ip_forward = 1

添加并保存规则

#DR
//配置第二张网卡
[root@DR ~]# vim /etc/sysconfig/network-Sncripts/ifcfg-ens192
TYPE=Ethernet
BOOTPROTO=static
ONBOOT=yes
NAME=ens192
DEVICE=ens192
IPADDR=192.168.222.250
PREFIX=24

[root@DR ~]# systemctl restart NetworkManager
[root@DR ~]# ifdown ens192;ifup ens192
[root@DR ~]# ip a
······
3: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:56:9e:9c brd ff:ff:ff:ff:ff:ff
    inet 192.168.222.250/24 brd 192.168.11.255 scope global noprefixroute ens192
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe56:9e9c/64 scope link 
       valid_lft forever preferred_lft forever

//添加调度器
[root@DR ~]# ipvsadm -A -t 192.168.222.250:80 -s rr

//添加跳转的IP地址
[root@DR ~]# ipvsadm -a -t 192.168.222.250:80 -r 192.168.100.4:80 -m
[root@DR ~]# ipvsadm -a -t 192.168.222.250:80 -r 192.168.100.5:80 -m
[root@DR ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.222.250:80 rr
  -> 192.168.100.4:80             Masq    1      0          0         
  -> 192.168.100.5:80             Masq    1      0          0    
  
//保存规则
[root@DR ~]# ipvsadm -Sn > /etc/sysconfig/ipvsadm
[root@DR ~]# systemctl enable --now ipvsadm

访问测试

#client
[root@client ~]# curl 192.168.222.250
lvs-web2
[root@client ~]# curl 192.168.222.250
lvs-web1
[root@client ~]# curl 192.168.222.250
lvs-web2
[root@client ~]# curl 192.168.222.250
lvs-web1
[root@client ~]# curl 192.168.222.250
lvs-web2
[root@client ~]# curl 192.168.222.250
lvs-web1

搭建NAT模式的HTTPS负载集群

在以上配置基础下搭建https

LVS服务器搭建CA服务端

生成一对密钥

#DR
[root@DR ~]# mkdir -p /etc/pki/CA/private
[root@DR ~]# cd /etc/pki/CA
[root@DR CA]# (umask 077;openssl genrsa -out private/cakey.pem 2048)
[root@DR CA]# openssl rsa -in private/cakey.pem -pubout

生成自签署证书

#DR
[root@DR CA]# openssl req -new -x509 -key private/cakey.pem -out cacert.pem -days 1024
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:HB
Locality Name (eg, city) [Default City]:HW
Organization Name (eg, company) [Default Company Ltd]:baozi  
Organizational Unit Name (eg, section) []:baozi
Common Name (eg, your name or your server's hostname) []:baozi
Email Address []:1@2.com

[root@DR CA]# touch index.txt && echo 01 > serial

RS1生成证书签署请求，并发送给CA

#RS1
[root@RS1 ~]# yum -y install mod_ssl
[root@RS1 ~]# mkdir /etc/httpd/ssl
[root@RS1 ~]# cd /etc/httpd/ssl
[root@RS1 ssl]# (umask 077;openssl genrsa -out httpd.key 2048)
[root@RS1 ssl]# openssl req -new -key httpd.key -days 1024 -out httpd.csr
Ignoring -days; not generating a certificate
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:HB
Locality Name (eg, city) [Default City]:WH
Organization Name (eg, company) [Default Company Ltd]:baozi
Organizational Unit Name (eg, section) []:baozi
Common Name (eg, your name or your server's hostname) []:baozi
Email Address []:1@2.com

Please enter the following 'extra' attributes
to be sent with your certificate request
A challenge password []:
An optional company name []:
[root@RS1 ssl]# ls
httpd.csr  httpd.key

#把证书签署请求文件发送给CA
[root@RS1 ssl]# scp httpd.csr root@192.168.100.3:/root/

CA签署证书并发给RS1

#DR
[root@DR ~]# mkdir /etc/pki/CA/newcerts
[root@DR ~]# touch /etc/pki/CA/index.txt

//跟踪最后一次颁发证书的序列号
[root@DR ~]# echo "01" > /etc/pki/CA/serial

[root@DR ~]# ls
anaconda-ks.cfg  httpd.csr
[root@DR ~]# openssl ca -in httpd.csr -out httpd.crt -days 1024
Using configuration from /etc/pki/tls/openssl.cnf
Check that the request matches the signature
Signature ok
Certificate Details:
        Serial Number: 1 (0x1)
        Validity
            Not Before: May  6 09:02:00 2021 GMT
            Not After : Feb 24 09:02:00 2024 GMT
        Subject:
            countryName               = CN
            stateOrProvinceName       = HB
            organizationName          = baozi
            organizationalUnitName    = baozi
            commonName                = baozi
            emailAddress              = 1@2.com
        X509v3 extensions:
            X509v3 Basic Constraints: 
                CA:FALSE
            Netscape Comment: 
                OpenSSL Generated Certificate
            X509v3 Subject Key Identifier: 
                C7:3B:A3:CD:87:98:12:12:CC:88:1A:ED:23:66:97:8A:66:EB:65:29
            X509v3 Authority Key Identifier: 
                keyid:CD:31:DC:BD:F4:70:26:6A:EA:AA:B1:83:08:8E:E6:FB:AD:F7:0B:BA

Certificate is to be certified until Feb 24 09:02:00 2024 GMT (1024 days)
Sign the certificate? [y/n]:y


1 out of 1 certificate requests certified, commit? [y/n]y
Write out database with 1 new entries
Data Base Updated
[root@DR ~]# ls
anaconda-ks.cfg  httpd.crt  httpd.csr

//CA把签署好的证书httpd.crt和服务端的证书cacert.pem发给RS1
[root@DR ~]# scp httpd.crt root@192.168.100.4:/etc/httpd/ssl  
[root@DR ~]# scp /etc/pki/CA/cacert.pem root@192.168.100.4:/etc/httpd/ssl

配置https

将RS1的证书和密钥发给RS2

#RS2
[root@RS2 ~]# yum -y install mod_ssl
[root@RS2 ~]# mkdir /etc/httpd/ssl

#RS1
[root@RS1 ~]# cd /etc/httpd/ssl/
[root@RS1 ssl]# scp cacert.pem httpd.crt httpd.key root@192.168.100.5:/etc/httpd/ssl

#RS2
[root@RS2 ~]# ls /etc/httpd/ssl/
cacert.pem  httpd.crt  httpd.key

修改https配置文件

#RS1
[root@RS1 ~]# vim /etc/httpd/conf.d/ssl.conf
#修改后如下所示
SSLCertificateFile /etc/httpd/ssl/httpd.crt
······
SSLCertificateKeyFile /etc/httpd/ssl/httpd.key
······
SSLCACertificateFile /etc/httpd/ssl/cacert.pem
······

//重启服务
[root@RS1 ~]# systemctl restart httpd
[root@RS1 ~]# ss -antl
State      Recv-Q     Send-Q          Local Address:Port           Peer Address:Port     
LISTEN     0          128                   0.0.0.0:22                  0.0.0.0:*        
LISTEN     0          128                      [::]:22                     [::]:*        
LISTEN     0          128                         *:443                       *:*        
LISTEN     0          128                         *:80                        *:*  

#RS2
[root@RS2 ~]# vim /etc/httpd/conf.d/ssl.conf
#修改后如下所示
SSLCertificateFile /etc/httpd/ssl/httpd.crt
······
SSLCertificateKeyFile /etc/httpd/ssl/httpd.key
······
SSLCACertificateFile /etc/httpd/ssl/cacert.pem
······

//重启服务
[root@RS2 ~]# systemctl restart httpd
[root@RS2 ~]# ss -antl
State      Recv-Q     Send-Q          Local Address:Port           Peer Address:Port     
LISTEN     0          128                   0.0.0.0:22                  0.0.0.0:*        
LISTEN     0          128                      [::]:22                     [::]:*        
LISTEN     0          128                         *:443                       *:*        
LISTEN     0          128                         *:80                        *:*  

添加并保存规则

#DR
//添加调度器
[root@DR ~]# ipvsadm -A -t 192.168.222.250:443 -s rr

//添加跳转的IP地址
[root@DR ~]# ipvsadm -a -t 192.168.222.250:443 -r 192.168.100.4 -m
[root@DR ~]# ipvsadm -a -t 192.168.222.250:443 -r 192.168.100.5 -m
[root@DR ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.222.250:80 rr
  -> 192.168.100.4:80             Masq    1      0          0         
  -> 192.168.100.5:80             Masq    1      0          0         
TCP  192.168.222.250:443 rr
  -> 192.168.100.4:443            Masq    1      0          0         
  -> 192.168.100.5:443            Masq    1      0          0   
  
//保存规则
[root@DR ~]# ipvsadm -Sn > /etc/sysconfig/ipvsadm

访问测试

#client
[root@client ~]# curl -k https://192.168.222.250
lvs-web2
[root@client ~]# curl -k https://192.168.222.250
lvs-web1
[root@client ~]# curl -k https://192.168.222.250
lvs-web2
[root@client ~]# curl -k https://192.168.222.250
lvs-web1
[root@client ~]# curl -k https://192.168.222.250
lvs-web2
[root@client ~]# curl -k https://192.168.222.250
lvs-web1

搭建DR模式的HTTP负载集群

DR模式是通过director将报文源和目标MAC地址修改，发送给RS，RS将响应报文直接发送给client。

本次环境为：

环境	DIP	VIP	需要安装的应用	系统版本
client	192.168.100.2	/	/	RedHat 8
DR	192.168.100.3	192.168.100.250	ipvsadm	RedHat 8
RS1	192.168.100.4	192.168.100.250	httpd	RedHat 8
RS2	192.168.100.5	192.168.100.250	httpd	RedHat 8

准备工作：

网卡配置去掉网关，yum源使用本地源

#DR：
[root@DR ~]# systemctl disable --now firewalld
[root@DR ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@DR ~]# setenforce 0
[root@DR ~]# yum -y install ipvsadm
[root@DR ~]# ip addr add 192.168.100.250/32 dev ens160
[root@DR ~]# ip a
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:56:9e:92 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.3/24 brd 192.168.100.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever
    inet 192.168.100.250/32 scope global ens160
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe56:9e92/64 scope link 
       valid_lft forever preferred_lft forever

#RS1：
[root@RS1 ~]# systemctl disable --now firewalld
[root@RS1 ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@RS1 ~]# setenforce 0
[root@RS1 ~]# yum -y install httpd net-tools
[root@RS1 ~]# systemctl enable --now httpd
[root@RS1 ~]# echo lvs-web1 > /var/www/html/index.html

#RS2：
[root@RS2 ~]# systemctl disable --now firewalld
[root@RS2 ~]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
[root@RS2 ~]# setenforce 0
[root@RS2 ~]# yum -y install httpd net-tools
[root@RS2 ~]# systemctl enable --now httpd
[root@RS2 ~]# echo lvs-web2 > /var/www/html/index.html

RS服务器配置

#RS1
[root@RS1 ~]# vim /etc/sysctl.conf
# 在最后面插入如下两行
net.ipv4.conf.all.arp_ignore = 1	# 将对应网卡设置为只回应目标IP为自身接口地址的ARP请求
net.ipv4.conf.all.arp_announce = 2	# 将ARP请求的源IP设置为eth0上的IP，也就是RIP

[root@RS1 ~]# sysctl -p
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

#RS2
[root@RS2 ~]# vim /etc/sysctl.conf
# 在最后面插入如下两行
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

[root@RS2 ~]# sysctl -p
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

配置VIP

一定要先设置好内核参数在配置VIP，如果先配置VIP，VIP配置好后会立即通告给所有人，而修改内核参数就是为了不通告

#RS1
[root@RS1 ~]# ip addr add 192.168.100.250/32 dev ens160
[root@RS1 ~]# ip a
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:61:17:d4 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.4/24 brd 192.168.100.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever
    inet 192.168.100.250/32 scope global ens160
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe61:17d4/64 scope link 
       valid_lft forever preferred_lft forever

#RS2
[root@RS2 ~]# ip addr add 192.168.100.250/32 dev ens160
[root@RS2 ~]# ip a
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:24:c8:11 brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.5/24 brd 192.168.100.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever
    inet 192.168.100.250/32 scope global ens160
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe24:c811/64 scope link 
       valid_lft forever preferred_lft forever

配置路由信息

#RS1
[root@RS1 ~]# route add -host 192.168.100.250/32 dev ens160
[root@RS1 ~]# echo '192.168.100.250/32 via 192.168.100.3' > /etc/sysconfig/network-Sncripts/route-ens160

#RS2
[root@RS2 ~]# route add -host 192.168.100.250/32 dev ens160
[root@RS2 ~]# echo '192.168.100.250/32 via 192.168.100.3' > /etc/sysconfig/network-Sncripts/route-ens160

添加并保存规则

#DR
[root@DR ~]# ipvsadm -A -t 192.168.100.250:80 -s wrr
[root@DR ~]# ipvsadm -a -t 192.168.100.250:80 -r 192.168.100.4 -g
[root@DR ~]# ipvsadm -a -t 192.168.100.250:80 -r 192.168.100.5 -g
   
[root@DR ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.250:80 wrr
  -> 192.168.100.4:80             Route   1      0          0         
  -> 192.168.100.5:80             Route   1      0          0  
  
[root@DR ~]# ipvsadm -Sn > /etc/sysconfig/ipvsadm
[root@DR ~]# systemctl enable --now ipvsadm

client上访问测试

#client
[root@client ~]# curl 192.168.100.250
lvs-web2
[root@client ~]# curl 192.168.100.250
lvs-web1
[root@client ~]# curl 192.168.100.250
lvs-web2
[root@client ~]# curl 192.168.100.250
lvs-web1
[root@client ~]# curl 192.168.100.250
lvs-web2
[root@client ~]# curl 192.168.100.250
lvs-web1

搭建DR模式的HTTPS负载集群

在以上配置基础下搭建https

# RS上安装mod_ssl
[root@RS1 ~]# yum -y install mod_ssl
[root@RS2 ~]# yum -y install mod_ssl

# 这里就不做证书，使用默认的证书，重启服务查看443是否启动
[root@RS1 ~]# systemctl restart httpd
[root@RS2 ~]# systemctl restart httpd

# 443端口已经起来
[root@RS1 ~]# ss -antl
State       Recv-Q      Send-Q           Local Address:Port             Peer Address:Port 
LISTEN      0           128                    0.0.0.0:22                    0.0.0.0:*   
LISTEN      0           128                          *:80                          *:*   
LISTEN      0           128                       [::]:22                       [::]:*   
LISTEN      0           128                          *:443                         *:*  

[root@RS2 ~]# ss -antl
State       Recv-Q      Send-Q           Local Address:Port             Peer Address:Port 
LISTEN      0           128                    0.0.0.0:22                    0.0.0.0:*   
LISTEN      0           128                          *:80                          *:*   
LISTEN      0           128                       [::]:22                       [::]:*   
LISTEN      0           128                          *:443                         *:*  

添加并保存规则

#DR
[root@DR ~]# ipvsadm -C
[root@DR ~]# ipvsadm -A -t 192.168.100.250:443 -s wrr
[root@DR ~]# ipvsadm -a -t 192.168.100.250:443 -r 192.168.100.4 -g
[root@DR ~]# ipvsadm -a -t 192.168.100.250:443 -r 192.168.100.5 -g
[root@DR ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.250:443 wrr
  -> 192.168.100.4:443            Route   1      0          0         
  -> 192.168.100.5:443            Route   1      0          0 
  
[root@DR ~]# ipvsadm -Sn > /etc/sysconfig/ipvsadm

client上访问测试

#client
[root@client ~]# curl -k https://192.168.100.250
lvs-web2
[root@client ~]# curl -k https://192.168.100.250
lvs-web1
[root@client ~]# curl -k https://192.168.100.250
lvs-web2
[root@client ~]# curl -k https://192.168.100.250
lvs-web1
[root@client ~]# curl -k https://192.168.100.250
lvs-web2
[root@client ~]# curl -k https://192.168.100.250
lvs-web1