Keepalive之nginx调度架构

author：JevonWei
版权声明：原创作品

---

单主模式Keepalive之Nginx调度

实验目的：实现Nginx调度的高可用，当一台Nginx调度器故障时，启用备用的Nginx调度，在架构中，启用了kepalive节点状态通知脚本机制，当keepalive的状态由MASTER变为BACKUP或由BACKUP变为MASTER时，以邮件通知用户，通过在通知脚本文件中定义BACKUP状态的节点启用nginx服务，是为了防止因Nginx服务故障导致节点权重降低从而变为BACKUP状态，为了避免此种现象，故在转换为BACKUP状态时需启动一次Nginx服务

实验环境

keepalive-A 172.16.253.108
keepalive-B 172.16.253.105
Nginx-A     172.16.253.108
Nginx-B     172.16.253.105
LVS-RS1     172.16.250.127
LVS-RS2     172.16.253.193
VIP         172.16.253.150
client      172.16.253.177

环境拓扑图

web服务集群

为了更好的观察实验结果，故在此将RS1和RS2的web页面内容设置不一致，以致可以更清晰的区分RS1服务端和RS2服务端

LVS-RS1

[root@LVS-RS1 ~]# systemctl restart chronyd  \多台服务器时间同步
[root@LVS-RS1 ~]# iptables -F
[root@LVS-RS1 ~]# setenforce 0
[root@LVS-RS1 ~]# yum -y install nginx
[root@LVS-RS1 ~]# vim /usr/share/nginx/html/index.html 
<h1> Web RS1 </h1>
[root@LVS-RS1 ~]# systemctl start nginx

LVS-RS2

[root@LVS-RS2 ~]# systemctl restart chronyd  \多台服务器时间同步
[root@LVS-RS2 ~]# iptables -F
[root@LVS-RS2 ~]# setenforce 0
[root@LVS-RS2 ~]# yum -y install nginx
[root@LVS-RS2 ~]# vim /usr/share/nginx/html/index.html
<h1> Web RS2 </h1>
[root@LVS-RS2 ~]# systemctl start nginx

nginx调度集群

nginx-A

[root@nginx-A ~]# yum -y install nginx
[root@nginx-A ~]# vim /usr/share/nginx/html/index.html 
</h1> sorry from keepalive-A </h1>
[root@nginx-A ~]# vim /etc/nginx/nginx.conf
http {
    upstream websrvs {
        server 172.16.250.127:80;
        server 172.16.253.193:80;
    }
}
server {
    listen       80 default_server;
    location / {
        proxy_pass http://websrvs;
    }
}
[root@nginx-A ~]# systemctl start nginx

nginx-B

[root@nginx-B ~]# yum -y install nginx
[root@nginx-B ~]# vim /usr/share/nginx/html/index.html 
</h1> sorry from keepalive-B </h1>
[root@nginx-B ~]# vim /etc/nginx/nginx.conf
http {
    upstream websrvs {
        server 172.16.250.127:80;
        server 172.16.253.193:80;
    }
}
server {
    listen       80 default_server;
    location / {
        proxy_pass http://websrvs;
    }
}
[root@nginx-B ~]# systemctl start nginx

keepalive集群

keepalive-A

[root@keepalive-A ~]# yum -y install keepalived
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh  \定义状态检测通知脚本，当节点变为BACKUP时，启动nginx服务
#!/bin/bash 
    #
    contact='root@localhost'  \通知的联系者，即本地邮件服务器的通知邮件接受者

    notify() {
       local mailsubject="$(hostname) to be $1, vip floating" \指定邮件标题
       local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"  \指定邮件正文
       echo "$mailbody" | mail -s "$mailsubject" $contact  \发送通知信息邮件到邮件接收者
    }

    case $1 in
        master)
           notify master
           ;;
        backup)
            notify backup
            systemctl start nginx
            ;;      
        fault)
            notify fault    
            ;;
        *)
            echo "Usage: $(basename $0) {master|backup|fault}"
            exit 1
            ;;
        esac
[root@keepaliveA ~]# vim /etc/keepalived/keepalived.conf
global_defs {
    notification_email {
        jevon@danran.com
    }
    notification_email_from ka_admin@danran.com
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id keepaliveA
    vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down { \定义检测脚本
    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测
    interval 1 \每隔一秒检测
    weight -10
    fall 2 \检测两次故障则表示节点故障
    rise 1 \检测一次正常则表示节点正常
}
vrrp_script chk_nginx { \检测nginx服务是否运行脚本
    script "killall -0 nginx" \-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10
    interval 2
    weight -10
    fall 2
    rise 2
}

vrrp_instance VI_A {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQHuL
    }
    virtual_ipaddress {
    172.16.253.150/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    track_script { \脚本调用
        chk_mant_down
        chk_nginx \调用chk_nginx脚本
    }
}
[root@keepalive-A ~]# systemctl start keepalived
[root@keepalive-A ~]# systemctl status keepalived

keepalive-B

[root@keepalive-B ~]# yum -y install keepalived
[root@keepalive-A ~]# vim /etc/keepalived/notify.sh  \定义状态检测通知脚本
#!/bin/bash 
    #
    contact='root@localhost'  \通知的联系者，即本地邮件服务器的通知邮件接受者

    notify() {
       local mailsubject="$(hostname) to be $1, vip floating" \指定邮件标题
       local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"  \指定邮件正文
       echo "$mailbody" | mail -s "$mailsubject" $contact  \发送通知信息邮件到邮件接收者
    }

    case $1 in
        master)
           notify master
           ;;
        backup)
            notify backup
            systemctl start nginx
            ;;      
        fault)
            notify fault    
            ;;
        *)
            echo "Usage: $(basename $0) {master|backup|fault}"
            exit 1
            ;;
        esac
[root@keepalive-B ~]# vim /etc/keepalived/keepalived.conf
global_defs {
    notification_email {
        jevon@danran.com
    }
    notification_email_from ka_admin@danran.com
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id keepaliveA
    vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down { \定义检测脚本
    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0" \检测/etc/keepalived/down文件是否存在，若文件存在，则weight权重-10，若文件不存在，则每隔一秒检测
    interval 1 \每隔一秒检测
    weight -10
    fall 2 \检测两次故障则表示节点故障
    rise 1 \检测一次正常则表示节点正常
}
vrrp_script chk_nginx { \检测nginx服务是否运行脚本
    script "killall -0 nginx" \-0为探测信号，当nginx进程存在时，返回值$?=0,当nginx进程不存在时，返回值$?为非零，则权重减10
    interval 2
    weight -10
    fall 2
    rise 2
}

vrrp_instance VI_A {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQHuL
    }
    virtual_ipaddress {
    172.16.253.150/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    
    track_script { \脚本调用
        chk_mant_down
        chk_nginx \调用chk_nginx脚本
    }
}
[root@keepalive-B ~]# systemctl start keepalived
[root@keepalive-A ~]# systemctl start keepalived

测试

访问测试调度

[root@client ~]# for i in {1..10};do curl http://172.16.253.150;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>

测试检测监控Nginx服务资源的外部脚本

[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
    inet 172.16.253.150/32 scope global ens33
        valid_lft forever preferred_lft forever

手动停止Nginx服务，keepalive-A变为了BACKUP节点

[root@keepalive-A ~]# systemctl stop nginx
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff

几秒钟之后，Nginx服务再次自动启动，则Keepalive-A再次成为MASTER节点

[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
    inet 172.16.253.150/32 scope global ens33
        valid_lft forever preferred_lft forever

当nginx的80端口被占用时，Nginx服务停止之后不会再次启动，则keepalive-A变为BACKUP

[root@keepalive-A ~]# yum -y install httpd 
[root@keepalive-A ~]# systemctl stop nginx && systemctl start httpd \停止nginx的同时启动httpd
[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
[root@keepalive-A ~]# systemctl status keepalived.service 
Sep 05 21:21:14 danran.com Keepalived_vrrp[24153]: VRRP_Script(chk_nginx) failed

在HTTPD进程停止之后，80端口被释放，则keepalive-A上的Nginx也不会再次重启

双主模式的Nginx调度(一般不做会话绑定)

双主模式的目的是为了访问web访问路径的单点故障，当一个web虚拟路由故障时，可使用另外一个web路径访问，最终是为了实现集群的高可用性
实验环境

keepalive-A 172.16.253.108
keepalive-B 172.16.253.105
Nginx-A     172.16.253.108
Nginx-B     172.16.253.105
LVS-RS1     172.16.250.127
LVS-RS2     172.16.253.193
VI-A        172.16.253.150
VI-B        172.16.253.140
client      172.16.253.177

环境拓扑图

keepalive-A

[root@keepalive-A ~]# vim /etc/keepalived/notify.sh 
#!/bin/bash
#
contact='root@localhost'

notify() {
    local mailsubject="$(hostname) to be $1, vip floating"
    local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$mailbody" | mail -s "$mailsubject" $contact
}

case $1 in
master)
    notify master
    ;;
backup)
    notify backup
    systemctl start nginx
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac
[root@keepalive-A ~]# systemctl stop keepalived  
[root@keepalive-A ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
    jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down {
    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
    interval 1
    weight -10
    fall 2
    rise 1
}
vrrp_script chk_nginx {
    script "killall -0 nginx"
    interval 2
    weight -10
    fall 2
    rise 2
}
vrrp_instance VI_A {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQHuL
    }
    virtual_ipaddress {
        172.16.253.150/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    track_script {
        chk_mant_down
        chk_nginx
    }
}
vrrp_instance VI_B {
    state BACKUP
    interface ens33
    virtual_router_id 52
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQH99
    }
    virtual_ipaddress {
        172.16.253.140/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    track_script {
        chk_mant_down
        chk_nginx
    }
}
[root@keepalive-A ~]# systemctl start keepalived

keepalive-B

[root@keepalive-A ~]# vim /etc/keepalived/notify.sh 
#!/bin/bash
#
contact='root@localhost'

notify() {
    local mailsubject="$(hostname) to be $1, vip floating"
    local mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$mailbody" | mail -s "$mailsubject" $contact
}

case $1 in
master)
    notify master
    ;;
backup)
    notify backup
    systemctl start nginx
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac
[root@keepalive-B ~]# systemctl stop keepalived  
[root@keepalive-B ~]# vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
    jevon@danran.com
}
notification_email_from ka_admin@danran.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id keepaliveA
vrrp_mcast_group4 224.103.5.5
}
vrrp_script chk_mant_down {
    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
    interval 1
    weight -10
    fall 2
    rise 1
}
vrrp_script chk_nginx {
    script "killall -0 nginx"
    interval 2
    weight -10
    fall 2
    rise 2
}
vrrp_instance VI_A {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQHuL
    }
    virtual_ipaddress {
        172.16.253.150/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    track_script {
        chk_mant_down
        chk_nginx
    }
}
vrrp_instance VI_B {
    state MASTER
    interface ens33
    virtual_router_id 52
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass qr8hQH99
    }
    virtual_ipaddress {
        172.16.253.140/32
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    }
    track_script {
        chk_mant_down
        chk_nginx
    }
}
[root@keepalive-B ~]# systemctl start keepalived

查看keepalive-A和keepalive-B IP地址，确认虚拟路由配置正确

[root@keepalive-A ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:75:dc:3c brd ff:ff:ff:ff:ff:ff
    inet 172.16.253.150/32 scope global ens33
   valid_lft forever preferred_lft forever
[root@keepalive-B ~]# ip a l
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:0e:47:87 brd ff:ff:ff:ff:ff:ff
    inet 172.16.253.140/32 scope global ens33
   valid_lft forever preferred_lft forever

测试Nginx调度

[root@client ~]# for i in {1..10};do curl http://172.16.253.150;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
[root@client ~]# for i in {1..10};do curl http://172.16.253.140;done
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>
<h1> Web RS2 </h1>
<h1> Web RS1 </h1>