LVS——配置实例

NAT模式配置httpd & httpds

环境如下：

主机	ip	角色
调度器	192.168.197.141	dip
调度器	192.168.197.250	vip
Real server1	192.168.197.142	rip
Real server2	192.168.197.143	rip

配置步骤

• 为调度器配置vip地址

注：这里实例是在同一网卡上添加vip地址，如若需要，重新新增一网卡更切近生产环境

[root@localhost ~]# cd /etc/sysconfig/network-scripts/
[root@localhost network-scripts]# vim ifcfg-ens160 
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
NAME=ens160
DEVICE=ens160
ONBOOT=yes
IPADDR0=192.168.197.141               # IPADDR0为dip
NETMASK0=255.255.255.0
GATEWAY0=192.168.197.2
IPADDR1=192.168.197.250               # IPADDR1为vip
NETMASK1=255.255.255.0
GATEWAY1=192.168.197.2

• 为RS1配置路由 指向调度器的vip

[root@localhost ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens160 

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
NAME=ens160
DEVICE=ens160
ONBOOT=yes
IPADDR=192.168.197.142
NETMASK=255.255.255.0
GATEWAY=192.168.197.141          #默认网关指向调度器的dip

# route命令查看
[root@localhost yum.repos.d]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.197.141 0.0.0.0         UG    100    0        0 ens160
192.168.197.0   0.0.0.0         255.255.255.0   U     100    0        0 ens160

• 为RS2配置路由，指向调度器的vip

[root@localhost ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens160 

TYPE=Ethernet
BROWSER_ONLY=no
BOOTPROTO=static
NAME=ens160
DEVICE=ens160
ONBOOT=yes
IPADDR=192.168.197.143
NETMASK=255.255.255.0
GATEWAY=192.168.197.141        #默认网关指向调度器的dip


# route命令查看
[root@localhost yum.repos.d]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.197.141 0.0.0.0         UG    100    0        0 ens160
192.168.197.0   0.0.0.0         255.255.255.0   U     100    0        0 ens160

• 开启调度器的ip转换功能

[root@localhost yum.repos.d]# vim /etc/sysctl.conf
......
net.ipv4.ip_forward = 1

# sysctl读取配置文件
[root@localhost yum.repos.d]# sysctl -p
net.ipv4.ip_forward = 1

• ipvsadm配置调度器

[root@localhost yum.repos.d]# ipvsadm -A -t 192.168.197.250:80 -s rr   ## -t指定vip和集群服务端口，-r指定算法

# 查看lvs list
[root@localhost yum.repos.d]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.197.250:80 rr

• ipvsadm添加后端服务器

[root@localhost yum.repos.d]# ipvsadm -a -t 192.168.197.250:80 -r 192.168.197.142:80 -m     ## -a 添加RS，-t VIP,-r 指定rip, -m,指定lvs模式为NAT模式
[root@localhost yum.repos.d]# ipvsadm -a -t 192.168.197.250:80 -r 192.168.197.143:8080 -m
[root@localhost yum.repos.d]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.197.250:80 rr
  -> 192.168.197.142:80           Masq    1      0          0         
  -> 192.168.197.143:8080          Masq    1      0          0    

## list写入配置文件
[root@localhost yum.repos.d]# ipvsadm -Sn > /etc/sysconfig/ipvsadm
[root@localhost yum.repos.d]# cat /etc/sysconfig/ipvsadm
-A -t 192.168.197.250:80 -s rr
-a -t 192.168.197.250:80 -r 192.168.197.142:80 -m -w 1
-a -t 192.168.197.250:80 -r 192.168.197.143:8080 -m -w 1

• RS开启httpd服务

## RS1开启httpd服务
[root@localhost html]# systemctl enable --now httpd
Created symlink /etc/systemd/system/multi-user.target.wants/httpd.service → /usr/lib/systemd/system/httpd.service.
[root@localhost html]# ss -antl
State     Recv-Q     Send-Q          Local Address:Port         Peer Address:Port    
LISTEN    0          128                   0.0.0.0:22                0.0.0.0:*       
LISTEN    0          128                         *:80                      *:*       
LISTEN    0          128                      [::]:22                   [::]:*  

## RS2开启httpd服务
[root@localhost ~]# systemctl enable --now httpd
Created symlink /etc/systemd/system/multi-user.target.wants/httpd.service → /usr/lib/systemd/system/httpd.service.
[root@localhost ~]# vim /var/www/html/index.html
[root@localhost ~]# ss -antl
State     Recv-Q     Send-Q           Local Address:Port           Peer Address:Port     
LISTEN    0          128                    0.0.0.0:22                  0.0.0.0:*        
LISTEN    0          128                          *:80                        *:*        
LISTEN    0          128                       [::]:22                     [::]:*

# 修改RS2上httpd监控端口号
[root@localhost ~]# vim /etc/httpd/conf/httpd.conf 
......
#Listen 12.34.56.78:80
Listen 8080    # 修改为8080
......

• 调度器上访问验证

NAT模式配置HTTPS集群

• 在DR上生成一队密钥

[root@localhost ~]# cd /etc/pki/CA/
[root@localhost ~]# cd /etc/pki/CA/
[root@localhost CA]# mkdir private
[root@localhost CA]# (umask 077;openssl genrsa -out private/cakey.pem 2048)
Generating RSA private key, 2048 bit long modulus (2 primes)
...................................+++++
...........+++++
e is 65537 (0x010001)

• 生成自签署证书

[root@localhost CA]# openssl req -new -x509 -key private/cakey.pem -out cacert.pem -days 365
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:HB   
State or Province Name (full name) []:HB
Locality Name (eg, city) [Default City]:WH
Organization Name (eg, company) [Default Company Ltd]:test     
Organizational Unit Name (eg, section) []:test
Common Name (eg, your name or your server's hostname) []:test
Email Address []:121@qq.com

• RS1上生成证书签署请求，并发送给CA

[root@localhost html]# cd /etc/httpd/
[root@localhost httpd]# mkdir ssl
[root@localhost httpd]# cd ssl/
[root@localhost ssl]# (umask 077;openssl genrsa -out httpd.key 2048)
Generating RSA private key, 2048 bit long modulus (2 primes)
.....................................................................................................................................................+++++
................................+++++
e is 65537 (0x010001)

# 生成http.csr
[root@localhost ssl]# openssl req -new -key httpd.key -days 1024 -out httpd.csr
Ignoring -days; not generating a certificate
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CH^HN
string is too long, it needs to be no more than 2 bytes long
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:HB  
Locality Name (eg, city) [Default City]:WH
Organization Name (eg, company) [Default Company Ltd]:123
Organizational Unit Name (eg, section) []:123
Common Name (eg, your name or your server's hostname) []:123
Email Address []:121@qq.com   

Please enter the following 'extra' attributes
to be sent with your certificate request
A challenge password []:
An optional company name []:
[root@localhost ssl]# ls
httpd.csr  httpd.key

[root@localhost ssl]# scp httpd.csr root@192.168.197.141:/root

• DR上签署证书

[root@localhost ~]# openssl ca -in httpd.csr -out httpd.crt -days 365
Using configuration from /etc/pki/tls/openssl.cnf
Check that the request matches the signature
Signature ok
Certificate Details:
        Serial Number: 1 (0x1)
        Validity
            Not Before: Nov 11 11:18:28 2020 GMT
            Not After : Nov 11 11:18:28 2021 GMT
......
Data Base Updated
[root@localhost ~]# ls
anaconda-ks.cfg  httpd.crt  httpd.csr

• DR上将httpd.crt与cacert.pem发送给RS1和RS2

[root@localhost ~]# scp httpd.crt root@192.168.197.142:/etc/httpd/ssl
[root@localhost ~]# scp /etc/pki/CA/cacert.pem root@192.168.197.142:/etc/httpd/ssl

[root@localhost ~]# scp httpd.crt root@192.168.197.143:/etc/httpd/ssl
[root@localhost ~]# scp /etc/pki/CA/cacert.pem root@192.168.197.143:/etc/httpd/ssl

• RS1上将httpd.key发送给RS2

[root@localhost ssl]# scp httpd.key root@192.168.197.143:/etc/httpd/ssl
[root@localhost ssl]# yum install -y mod_ssl

• RS1和RS2上修改配置文件

[root@localhost conf.d]# vim /etc/httpd/conf.d/ssl.conf
# General setup for the virtual host, inherited from global configuration
DocumentRoot "/var/www/html"     # 此两行注释取消掉
ServerName www.example.com:443

......
SSLCertificateFile /etc/httpd/ssl/httpd.crt    # 指定证书和密钥的地址
SSLCertificateKeyFile /etc/httpd/ssl/httpd.key

• ipvsadm配置

[root@localhost ~]# ipvsadm -a -t 192.168.197.141:443 -r 192.168.197.142:443 -m
[root@localhost ~]# ipvsadm -a -t 192.168.197.141:443 -r 192.168.197.143:443 -m
[root@localhost ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.197.141:443 rr
  -> 192.168.197.142:443          Masq    1      0          0         
  -> 192.168.197.143:443          Masq    1      0          0         
TCP  192.168.197.250:80 rr
  -> 192.168.197.142:80           Masq    1      0          0         
  -> 192.168.197.143:8080         Masq    1      0          0   

• 调度器上访问认证

DR模式配置mysql集群

环境信息

主机	ip	角色
调度器	192.168.197.141	dip
调度器,RS1,RS2	192.168.197.250	vip
Real server1	192.168.197.142	rip
Real server2	192.168.197.143	rip
NFS服务器	192.168.197.137	共享mysql库

• 设置RS上的内核参数

[root@localhost conf.d]# vim /etc/sysctl.conf 
......
# For more information, see sysctl.conf(5) and sysctl.d(5).
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

## 重读配置文件
[root@localhost conf.d]# sysctl -p
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2

• 配置RS上的vip

## RS1
[root@localhost conf.d]# vim /etc/sysconfig/network-scripts/ifcfg-ens160 
TYPE=Ethernet
BOOTPROTO=static
NAME=ens160
DEVICE=ens160
ONBOOT=yes

IPADDR0=192.168.197.142
NETMASK0=255.255.255.0
GATEWAY0=192.168.197.2

IPADDR1=192.168.197.250           # 指定vip
NETMASK1=255.255.255.0

## RS2
[root@localhost conf.d]# vim /etc/sysconfig/network-scripts/ifcfg-ens160 
TYPE=Ethernet
BOOTPROTO=static
NAME=ens160
DEVICE=ens160
ONBOOT=yes

IPADDR0=192.168.197.143
NETMASK0=255.255.255.0
GATEWAY0=192.168.197.2

IPADDR1=192.168.197.250
NETMASK1=255.255.255.0

• 验证访问vip是否访问到DR上
  
  

• 调度器上配置ipvsadm

[root@localhost ~]# ipvsadm -A -t 192.168.197.250:3306 -s rr

[root@localhost ~]# ipvsadm -a -t 192.168.197.250:3306 -r 192.168.197.142:3306 -g
[root@localhost ~]# ipvsadm -a -t 192.168.197.250:3306 -r 192.168.197.143:3306 -g
[root@localhost ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.197.250:3306 rr
  -> 192.168.197.142:3306         Route   1      0          0         
  -> 192.168.197.143:3306         Route   1      0          0  

## 保存设置
[root@localhost ~]# ipvsadm -Sn > /etc/sysconfig/ipvsadm

• NFS服务器配置

[root@localhost lib]# vim /etc/exports
/var/lib/mysql 192.168.197.0/24(rw,anonuid=27,anongid=27)

# 共享文件的所属用户与组变更
[root@localhost lib]# chown -R mysql.mysql mysql/


# RS1挂载到nfs共享文件中
[root@localhost mysql]# mount -t nfs 192.168.197.137:/var/lib/mysql /var/lib/mysql/

[root@localhost mysql]# df -h
Filesystem                      Size  Used Avail Use% Mounted on
devtmpfs                        1.9G     0  1.9G   0% /dev
tmpfs                           2.0G     0  2.0G   0% /dev/shm
tmpfs                           2.0G  9.0M  2.0G   1% /run
tmpfs                           2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/mapper/rhel-root            17G  2.6G   15G  15% /
/dev/nvme0n1p1                 1014M  154M  861M  16% /boot
tmpfs                           392M     0  392M   0% /run/user/0
/dev/sr0                        7.4G  7.4G     0 100% /mnt
192.168.197.137:/var/lib/mysql   47G  1.8G   46G   4% /var/lib/mysql  # 成功挂载

• RS1初始化数据库并开启服务

[root@localhost mysql]# !vim
vim /etc/my.cnf 

[mysqld]
basedir=/usr/local/mysql
datadir=/var/lib/mysql
user=mysql
socket=/tmp/mysql.sock
port=3306
pid-file=/var/lib/mysql/mysql.pid
skip-name-resolve

## 初始化
[root@localhost mysql]# mysqld --initialize-insecure --user=mysql --datadir=/var/lib/mysql --explicit_defaults_for_timestamp
......

## 查看nfs共享文件中的内容
[root@localhost mysql]# ls
auto.cnf        ibdata1      ib_logfile1  performance_schema
ib_buffer_pool  ib_logfile0  mysql        sys

# 配置mysql，开启服务
[root@localhost mysql]# cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql
[root@localhost mysql]# vim /etc/init.d/mysqld 
[root@localhost mysql]# chkconfig --add mysqld
[root@localhost mysql]# service mysqld start
Starting MySQL.Logging to '/var/lib/mysql/localhost.localdomain.err'.
. SUCCESS! 

# 成功登录
[root@localhost mysql]# mysql
Welcome to the MySQL monitor.  Commands end with ; or g.
Your MySQL connection id is 2
Server version: 5.7.22 MySQL Community Server (GPL)

Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or 'h' for help. Type 'c' to clear the current input statement.

mysql>

• RS2配置mysql并将数据库存放文件目录挂载到nfs共享目录

## 必须安装nfs
[root@localhost local]# yum install -y nfs-utils

## 进行挂载
[root@localhost local]# mount -t nfs 192.168.197.137:/var/lib/mysql /var/lib/mysql/
[root@localhost local]# df -h
Filesystem                      Size  Used Avail Use% Mounted on
devtmpfs                        1.9G     0  1.9G   0% /dev
tmpfs                           2.0G     0  2.0G   0% /dev/shm
tmpfs                           2.0G  9.0M  2.0G   1% /run
tmpfs                           2.0G     0  2.0G   0% /sys/fs/cgroup
/dev/mapper/rhel-root            17G  4.6G   13G  28% /
/dev/sr0                        7.4G  7.4G     0 100% /mnt
/dev/nvme0n1p1                 1014M  154M  861M  16% /boot
tmpfs                           392M     0  392M   0% /run/user/0
192.168.197.137:/var/lib/mysql   47G  1.9G   46G   4% /var/lib/mysql
[root@localhost local]# cd mysql
[root@localhost mysql]# ls
bin  COPYING  docs  include  lib  man  README  share  support-files

## mysql配置
[root@localhost mysql]# vim /etc/my.cnf
[mysqld]
basedir=/usr/local/mysql
datadir=/var/lib/mysql
socket=/tmp/mysql.sock
user=mysql
port=3306
pid-file=/var/lib/mysql.pid
skip-name-resolve

## 为共享文件上级目录增加权限
[root@localhost mysql]# setfacl -m u:mysql:rwx /var/lib/
[root@localhost mysql]# ll -d /var/lib/mysql/

## 开启服务
[root@localhost mysql]# service mysqld stop
Shutting down MySQL.. SUCCESS!
[root@localhost mysql]# mysql
Welcome to the MySQL monitor.  Commands end with ; or g.
Your MySQL connection id is 2
Server version: 5.7.22 MySQL Community Server (GPL)

Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or 'h' for help. Type 'c' to clear the current input statement.

mysql> 

• 验证rs1与rs2是否同步数据

## rs1创建表
mysql> create table student(id int primary key not null,name varchar(100) not null);
Query OK, 0 rows affected (0.75 sec)

mysql> desc student;
+-------+--------------+------+-----+---------+-------+
| Field | Type         | Null | Key | Default | Extra |
+-------+--------------+------+-----+---------+-------+
| id    | int(11)      | NO   | PRI | NULL    |       |
| name  | varchar(100) | NO   |     | NULL    |       |
+-------+--------------+------+-----+---------+-------+
2 rows in set (0.04 sec)

## rs2查看表
mysql> show tables;
+------------------+
| Tables_in_sawyer |
+------------------+
| student          |
+------------------+
1 row in set (0.01 sec)

• 调度器上验证效果