Keepalived+Nginx实现高可用和双主节点负载均衡

原文：http://www.linuxidc.com/Linux/2013-10/90809.htm

简介

Nginx可以实现高并发反向代理，lvs集群可以实现负载均衡，但是他们都有一个共同的弊端，就是Nginx，lvs架构中Director是单点故障，有没有一个好的方案解决这个问题呢？答案是有。通过Keepalived就可以实现，前端Nginx，lvs中Director服务器的高可用和负载均衡，通过shell命令或者脚本可以实现对服务器状态和服务的监控！

一、环境介绍
1、系统环境及软件版本
操作系统：CentOS6.4-i386
软件版本：Nginx-1.4.2
keepalived-1.2.7
yum源：
# vim /etc/yum.repos.d/centos6.repo
[base] 
name=centos-base 
baseurl=http://mirrors.sohu.com/centos/$releasever/os/$basearch 
gpgcheck=1 
enable=1 
gpgkey=http://mirrors.sohu.com/centos/RPM-GPG-KEY-CentOS-6
[epel] 
name=Fedora-epel 
baseurl=http://mirrors.sohu.com/fedora-epel/$releasever/$basearch/ 
enable=1 
gpgcheck=0

提示：如果你的系统是centos，系统默认的yum源是可以用的
2、拓扑图

3、IP地址规划
Client: 172.16.254.28
Keepalived+Nginx1: 172.16.3.3 Vip: 172.16.3.100
Keepalived+Nginx2: 172.16.3.4 Vip: 172.16.3.200

二、安装
1、安装keepalived
[root@node1 ~]# yum install keepalived

2、编译安装Nginx

[root@node1 ~]#useradd -r nginx 
[root@node1 ~]#yum -y groupinstall "Development tools" "Server  Platform Development" 
[root@node1 ~]#yum -y install pcre-devel 
[root@node1 ~]#tar xf nginx-1.4.2.tar.gz 
[root@node1 ~]#cd nginx-1.4.2 
[root@node1 nginx-1.4.2]# ./configure  
--prefix=/usr 
--sbin-path=/usr/sbin/nginx 
--conf-path=/etc/nginx/nginx.conf  
--error-log-path=/var/log/nginx/error.log  
--http-log-path=/var/log/nginx/access.log  
--pid-path=/var/run/nginx/nginx.pid   
--lock-path=/var/lock/nginx.lock  
--user=nginx  
--group=nginx  
--with-http_ssl_module  
--with-http_flv_module  
--with-http_stub_status_module  
--with-http_gzip_static_module  
--http-client-body-temp-path=/var/tmp/nginx/client/ 
--http-proxy-temp-path=/var/tmp/nginx/proxy/ 
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ 
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi 
--http-scgi-temp-path=/var/tmp/nginx/scgi 
--with-pcre

提示：在两台服务器上都要安装Nginx
3、提供Nginx服务启动脚本

[root@node1 nginx-1.4.2]# vim /etc/rc.d/init.d/nginx 
#!/bin/sh 
# 
# nginx - this script starts and stops the nginx daemon 
# 
# chkconfig:  - 85 15 
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse  
#              proxy and IMAP/POP3 proxy server 
# processname: nginx 
# config:      /etc/nginx/nginx.conf 
# config:      /etc/sysconfig/nginx 
# pidfile:    /var/run/nginx.pid 
# Source function library. 
. /etc/rc.d/init.d/functions
# Source networking configuration. 
. /etc/sysconfig/network
# Check that networking is up. 
[ "$NETWORKING"= "no"] && exit0 
nginx="/usr/sbin/nginx"
prog=$(basename$nginx) 
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() { 
# make required directories 
user=`nginx -V 2>&1 | grep"configure arguments:"| sed's/[^*]*--user=([^ ]*).*/1/g'-` 
options=`$nginx -V 2>&1 | grep'configure arguments:'` 
foropt in$options; do
if[ `echo$opt | grep'.*-temp-path'` ]; then
value=`echo$opt | cut-d "="-f 2` 
if[ ! -d "$value"]; then
# echo "creating" $value 
mkdir-p $value && chown-R $user $value 
fi
fi
done
} 
start() { 
[ -x $nginx ] || exit5 
[ -f $NGINX_CONF_FILE ] || exit6 
make_dirs 
echo-n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE 
retval=$? 
echo
[ $retval -eq0 ] && touch$lockfile 
return$retval 
} 
stop() { 
echo-n $"Stopping $prog: "
killproc $prog -QUIT 
retval=$? 
echo
[ $retval -eq0 ] && rm-f $lockfile 
return$retval 
} 
restart() { 
configtest || return$? 
stop 
sleep1 
start 
} 
reload() { 
configtest || return$? 
echo-n $"Reloading $prog: "
killproc $nginx -HUP 
RETVAL=$? 
echo
} 
force_reload() { 
restart 
} 
configtest() { 
$nginx -t -c $NGINX_CONF_FILE 
} 
rh_status() { 
status $prog 
} 
rh_status_q() { 
rh_status >/dev/null2>&1 
} 
case"$1"in
start) 
rh_status_q && exit0 
$1 
;; 
stop) 
rh_status_q || exit0 
$1 
;; 
restart|configtest) 
$1 
;; 
reload) 
rh_status_q || exit7 
$1 
;; 
force-reload) 
force_reload 
;; 
status) 
rh_status 
;; 
condrestart|try-restart) 
rh_status_q || exit0 
;; 
*) 
echo$"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit2 
esac
[root@node1 nginx-1.4.2]# chmod +x /etc/rc.d/init.d/nginx 
[root@node1 nginx-1.4.2]# chkconfig --add nginx 
[root@node1 nginx-1.4.2]# chkconfig nginx on 
[root@node1 nginx-1.4.2]# service nginx start 
Starting nginx:                                            [  OK  ]

提示：服务脚本在两台Nginx服务器上都要提供

三、实现高可用

1、修改配置keepalived
MASTER:172.16.3.3
[root@node1 ~]# cd /etc/keepalived 
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak    # 备份keepalived配置文件 
[root@node1 keepalived]#vim keepalived.conf                    # 创建配置文件，如下 
! Configuration File for keepalived 
global_defs { 
  notification_email { 
    root@localhost 
  } 
  notification_email_from kaadmin@localhost 
  smtp_server 127.0.0.1 
  smtp_connect_timeout 30 
  router_id LVS_DEVEL 
} 
vrrp_script chk_nginx { 
    script "killall -0 nginx"
    interval 1 
    weight 2 
} 
vrrp_instance VI_10 { 
    state MASTER 
    interface eth0 
    virtual_router_id 131 
    priority 100 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.100 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify.sh master"
  notify_backup "/etc/keepalived/notify.sh backup"
  notify_fault "/etc/keepalived/notify.sh fault"
}

BACKUP:172.16.3.4

[root@node2 ~]# cd /etc/keeplived 
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak 
[root@node2 keepalived]# vim keepalived.conf 
! Configuration File for keepalived 
global_defs { 
  notification_email { 
    root@localhost 
  } 
  notification_email_from kaadmin@localhost 
  smtp_server 127.0.0.1 
  smtp_connect_timeout 30 
  router_id LVS_DEVEL 
} 
vrrp_script chk_nginx { 
  script "killall -0 nginx"
  interval 1 
  weight 2 
} 
vrrp_instance VI_10 { 
    state BACKUP 
    interface eth0 
    virtual_router_id 131 
    priority 99 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.100 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify.sh master"
  notify_backup "/etc/keepalived/notify.sh backup"
  notify_fault "/etc/keepalived/notify.sh fault"
}

2、服务监控脚本

[root@node1 ~]# vim /etc/keepalived/notify.sh 
#!/bin/bash 
# 
vip=172.16.3.100 
contact='root@localhost'
notify() { 
    mailsubject="`hostname` to be $1: $vip floating"
    mailbody="`date '+%F %H:%M:%S'`: vrrp transition,`hostname` changed to be $1"
    echo $mailbody | mail -s "$mailsubject" $contact 
} 
case "$1" in
  master) 
    notify master 
    /etc/rc.d/init.d/nginx start 
    exit 0 
  ;; 
  backup) 
    notify backup 
    /etc/rc.d/init.d/nginx restart 
    exit 0 
  ;; 
  fault) 
    notify fualt 
    exit 0 
  ;; 
  *) 
    echo 'Usage: `basename $0` {master|backup|fault}'
    exit 1 
  ;; 
esac

复制脚本到另一节点

[root@node1 ~]# scp -p /etc/keepalived/notify.sh root@172.16.3.4:/etc/keepalived/

3、验证

[root@node1 ~]# ip addr show 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 
    inet 127.0.0.1/8 scope host lo 
    inet6 ::1/128 scope host 
      valid_lft forever preferred_lft forever 
2:  eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000 
    link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff 
    inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0 
    inet 172.16.3.100/32 scope global eth0          # Vip此时在Keepalived+Nginx1上 
    inet6 fe80::20c:29ff:fe65:a02f/64 scope link 
      valid_lft forever preferred_lft forever

然后关闭Keepalived+Nginx1上的keepalived服务Vip会跳转到Keepalived上

四、实现负载均衡
1、修改Keepalived配置文件
Keepalived+Nginx1：
[root@node1 ~]# cd /etc/keepalived 
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak    # 备份keepalived配置文件 
[root@node1 keepalived]#vim keepalived.conf                        # 创建配置文件，如下 
! Configuration File for keepalived 
global_defs { 
  notification_email { 
    root@localhost 
  } 
  notification_email_from kaadmin@localhost 
  smtp_server 127.0.0.1 
  smtp_connect_timeout 30 
  router_id LVS_DEVEL 
} 
vrrp_script chk_nginx { 
    script "killall -0 nginx"
    interval 1 
    weight 2 
} 
vrrp_instance VI_10 { 
    state MASTER 
    interface eth0 
    virtual_router_id 131 
    priority 100 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.100 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify.sh master"
  notify_backup "/etc/keepalived/notify.sh backup"
  notify_fault "/etc/keepalived/notify.sh fault"
} 
vrrp_instance VI_20 { 
    state BACKUP 
    interface eth0 
    virtual_router_id 132 
    priority 99 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.200 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify1.sh master"
  notify_backup "/etc/keepalived/notify1.sh backup"
  notify_fault "/etc/keepalived/notify1.sh fault"
        auth_pass 123456 
 }

Keepalived+Nginx2
[root@node2 ~]# cd /etc/keeplived 
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak 
[root@node2 keepalived]# vim keepalived.conf 
! Configuration File for keepalived 
global_defs { 
  notification_email { 
    root@localhost 
  } 
  notification_email_from kaadmin@localhost 
  smtp_server 127.0.0.1 
  smtp_connect_timeout 30 
  router_id LVS_DEVEL 
} 
vrrp_script chk_nginx { 
  script "killall -0 nginx"
  interval 1 
  weight 2 
} 
vrrp_instance VI_10 { 
    state BACKUP 
    interface eth0 
    virtual_router_id 131 
    priority 99 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.100 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify.sh master"
  notify_backup "/etc/keepalived/notify.sh backup"
  notify_fault "/etc/keepalived/notify.sh fault"
} 
vrrp_instance VI_20 { 
    state MASTER 
    interface eth0 
    virtual_router_id 132 
    priority 100 
    advert_int 1 
    authentication { 
        auth_type PASS 
        auth_pass 123456 
    } 
    virtual_ipaddress { 
    172.16.3.200 
    } 
    track_script { 
        chk_nginx 
  } 
  notify_master "/etc/keepalived/notify1.sh master"
  notify_backup "/etc/keepalived/notify1.sh backup"
  notify_fault "/etc/keepalived/notify1.sh fault"
}

提示：notify1.sh脚本与notify.sh只修改了vip
2、重启Keepalived服务
[root@node1 ~]# service keepalived restart 
[root@node2 ~]# service keepalived restart

3、验证

[root@node2 ~]# ip addr show 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 
    inet 127.0.0.1/8 scope host lo 
    inet6 ::1/128 scope host 
      valid_lft forever preferred_lft forever 
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000 
    link/ether 00:0c:29:52:ee:44 brd ff:ff:ff:ff:ff:ff 
    inet 172.16.3.4/16 brd 172.16.255.255 scope global eth0 
    inet 172.16.3.200/32 scope global eth0    #此时vip2在Keepalived+Nginx2上 
    inet6 fe80::20c:29ff:fe52:ee44/64 scope link 
      valid_lft forever preferred_lft forever

4、关闭Keepalived+Nginx2上的Keepalived服务模拟故障，vip2会转移到Keepalived+Nginx1上，反之依然

[root@node1 ~]# ip addr show 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 
    inet 127.0.0.1/8 scope host lo 
    inet6 ::1/128 scope host 
      valid_lft forever preferred_lft forever 
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000 
    link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff 
    inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0 
    inet 172.16.3.100/32 scope global eth0 
    inet 172.16.3.200/32 scope global eth0 
    inet6 fe80::20c:29ff:fe65:a02f/64 scope link 
      valid_lft forever preferred_lft forever