postgresql 高可用 etcd + patroni 之八 haproxy + keepalived

os: ubuntu 16.04
postgresql: 9.6.9
etcd: 3.2.18
patroni: 1.4.4
haproxy: 1.6.3
keepalived: 1.2.24

ip规划
192.168.56.101 node1 master
192.168.56.102 node2 slave
192.168.56.103 node3 slave
192.168.56.104 node4 slave
192.168.56.105 node5 slave

192.168.56.201 haproxy1
192.168.56.202 haproxy2

192.168.56.210 keepalived vip
192.168.56.211 keepalived vip
192.168.56.212 keepalived vip

keepalived 是 ha 的一把利器，应用范围极广，这里主要用来对 haproxy 做高可用。关于 keepalived 的更多知识，可以自行baidu。

安装、配置好haproxy

haproxy1、haproxy2节点都需要做
详细过程略，参考上一篇blog

安装 keepalived

# apt install keepalived
Reading package lists... Done
Building dependency tree       
Reading state information... Done
The following additional packages will be installed:
  ipvsadm libnl-route-3-200 libsensors4 libsnmp-base libsnmp30
Suggested packages:
  heartbeat ldirectord lm-sensors snmp-mibs-downloader
The following NEW packages will be installed:
  ipvsadm keepalived libnl-route-3-200 libsensors4 libsnmp-base libsnmp30
0 upgraded, 6 newly installed, 0 to remove and 171 not upgraded.
Need to get 1,427 kB of archives.
After this operation, 5,570 kB of additional disk space will be used.
Do you want to continue? [Y/n] 

去除了 /usr/share/snmp、/usr/share/doc、/usr/share/man

# dpkg -L keepalived
/.
/usr
/usr/lib
/usr/lib/systemd
/usr/lib/systemd/system
/usr/lib/systemd/system/keepalived.service
/usr/bin
/usr/bin/genhash
/usr/share
/usr/sbin
/usr/sbin/keepalived
/lib
/lib/systemd
/lib/systemd/system
/lib/systemd/system/keepalived.service
/etc
/etc/keepalived
/etc/default
/etc/default/keepalived
/etc/init.d
/etc/init.d/keepalived

查看 keepalived.service

# more /lib/systemd/system/keepalived.service 
[Unit]
Description=Keepalive Daemon (LVS and VRRP)
After=network-online.target
Wants=network-online.target
# Only start if there is a configuration file
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf

[Service]
Type=forking
KillMode=process
PIDFile=/var/run/keepalived.pid
# Read configuration variable file if it is present
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived $DAEMON_ARGS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target

查看 /etc/init.d/keepalived

# more /etc/init.d/keepalived

#! /bin/sh
#
# keepalived   LVS cluster monitor daemon.
#
#              Written by Andres Salomon <dilinger@voxel.net>
#
### BEGIN INIT INFO
# Provides:          keepalived
# Required-Start:    $syslog $network $remote_fs
# Required-Stop:     $syslog $network $remote_fs
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Starts keepalived
# Description:       Starts keepalived lvs loadbalancer
### END INIT INFO
PATH=/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/sbin/keepalived
NAME=keepalived
DESC=keepalived
CONFIG=/etc/keepalived/keepalived.conf
TMPFILES="/tmp/.vrrp /tmp/.healthcheckers"

#includes lsb functions 
. /lib/lsb/init-functions

test -f $CONFIG || exit 0 
test -f $DAEMON || exit 0

# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME

case "$1" in
  start)
	log_daemon_msg "Starting $DESC" "$NAME"
	for file in $TMPFILES
	do 
       		test -e $file && test ! -L $file && rm $file
	done
	if start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid 
               --exec $DAEMON -- $DAEMON_ARGS; then
		log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  stop)
	log_daemon_msg "Stopping $DESC" "$NAME"
	if start-stop-daemon --oknodo --stop --quiet --pidfile /var/run/$NAME.pid 
        	--exec $DAEMON; then
		log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  reload|force-reload)
	log_action_begin_msg "Reloading $DESC configuration..."
      	if start-stop-daemon --stop --quiet --signal 1 --pidfile 
               /var/run/$NAME.pid --exec $DAEMON; then
	       log_end_msg 0
	else
		log_action_end_msg 1
	fi
       ;;
  restart)
  	log_action_begin_msg "Restarting $DESC" "$NAME"

       	start-stop-daemon --stop --quiet --pidfile 
		/var/run/$NAME.pid --exec $DAEMON || true 
       sleep 1
       if start-stop-daemon --start --quiet --pidfile 
               /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_ARGS; then
	       log_end_msg 0
	else
		log_end_msg 1
	fi
       ;;
  *)
       echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|force-reload}" >&2
       exit 1
       ;;
esac

exit 0

配置 keepalived

注意：可以修改 state为 MASTER、BACKUP

# vi /etc/keepalived/keepalived.conf


! Configuration File for keepalived    #! 表示注释
global_defs {
        router_id LVS_HAProxy1                  # 路由标示，相同局域网内唯一
}

vrrp_script chk_health {
        script "/etc/keepalived/scripts/health_check.sh"   # 监控脚本
        interval 5   # 每隔5s探测一次
        fall 5       # 连续检测5次失败后，则进行切换IP
        rise 5       # 5次检测成功后将OK

}


vrrp_instance VI_HAProxy {                      # 定义VRRP热备实例
        state MASTER                            # 热备状态，MASTER(主)；SLAVE(从)；BACKUP(备份);只能有一个MASTER
        interface eno1                          # 承载VIP地址的物理接口
        virtual_router_id 51                    # 虚拟路由器的ID号，组内保存一致
        priority 100                            # 优先级,数值越大优先级越高。
        advert_int 2                            # 主备之间的通告间隔秒数（心跳频率）
        nopreempt                               # 不抢占，只在优先级高的机器上设置即可，优先级低的机器不设置
        authentication {                        # 认证信息，每个热备组保持一致
                auth_type PASS                  # 认证类型，主备切换时的验证
                auth_pass 1qaz                  # 密码字串
        }
        track_script {
                chk_health
        }
        virtual_ipaddress {                     # 指定漂移地址（VIP），可以有多个
                192.168.56.210/24 dev eno1 label eno1:1
                192.168.56.211/24 dev eno1 label eno1:2
                192.168.56.212/24 dev eno1 label eno1:3
        } 
        notify_master /etc/keepalived/scripts/lvs_notify_master.sh     # 切换成为主时的通知
        notify_backup /etc/keepalived/scripts/lvs_notify_backup.sh     # 切换成为备时的通知
        notify_fault  /etc/keepalived/scripts/lvs_notify_fault.sh      # 调用失败时的通知
}

/etc/keepalived/scripts/health_check.sh

#!/bin/bash
# 检查自己是否可以成为主 0:可以 大于0:不可以

IP_GATEWAY="192.168.56.1"
IP_LIST=$IP_GATEWAY" 192.168.56.201 192.168.56.202"

logfile="/var/log/keepalived_lvs_status.log"
log() {
	now=`date`
	content=$1
	echo  ${now}":"${content}  >>$logfile
}

ping_status() {
    IP=$1
    for ((i=1;i<=3;i++)); do
        if ping -c 1 -W 1 $IP >/dev/null; then
            #log "$IP Ping is successful."
            return 0
        else
            log "$IP Ping is failure " 
        fi
    done
    #三次都没有ping通，按失败处理。
    if [ "$IP" = "$IP_GATEWAY" ] ;then
        return 99
    else
        return 1
    fi
}

log '---------- health check start ...---------'

FAIL_COUNT=0
for IP in $IP_LIST; do
    log "ping $IP ..."
    ping_status $IP
    ret=$?
    FAIL_COUNT=$(($FAIL_COUNT+$ret))
done

log "fail ping num:$FAIL_COUNT"


if [ $FAIL_COUNT -gt 2 ]; then
    log "fail ping more than 2,exit 1"
    exit 1	
fi

status=$(systemctl status haproxy|grep Active|grep running|wc -l)
if [ "${status}" = "0" ]; then
    log 'haproxy is not running, restart it'
    service haproxy restart

    status2=$(systemctl status haproxy|grep Active|grep running|wc -l)
    if [ "${status2}" = "0"  ]; then
        log 'haproxy is not running, exit 1'
        exit 1
    fi
else
    log 'haproxy is running'
fi

log 'check ok,exit 0'
exit 0

/etc/keepalived/scripts/lvs_notify_master.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_master.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to master'

/etc/keepalived/scripts/lvs_notify_backup.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_backup.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to backup'

/etc/keepalived/scripts/lvs_notify_fault.sh

#!/bin/bash
# 调用公司内部的短信接口通知或者微信通知，邮件通知时效性较差

logfile="/var/log/keepalived_lvs_fault.log"
log() {
        now=`date`
        content=$1
        echo  ${now}":"${content}  >>$logfile
}

log 'keepalived to fault'

验证 keepalived

# systemctl start keepalived
# systemctl enable keepalived
# ps -ef|grep -i keep
root     16568     1  0 10:34 ?        00:00:00 /usr/sbin/keepalived
root     16569 16568  0 10:34 ?        00:00:00 /usr/sbin/keepalived
root     16570 16568  0 10:34 ?        00:00:00 /usr/sbin/keepalived

# ip a show eno1
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:7d:f5:f0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.101/24 brd 192.168.56.255 scope global eno1
       valid_lft forever preferred_lft forever
    inet 192.168.56.210/24 scope global secondary eno1:1
       valid_lft forever preferred_lft forever
    inet 192.168.56.211/24 scope global secondary eno1:2
       valid_lft forever preferred_lft forever
    inet 192.168.56.212/24 scope global secondary eno1:3
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe7d:f5f0/64 scope link 
       valid_lft forever preferred_lft forever
	   

chk_health功能验证

# systemctl stop haproxy

# tail -f /var/log/keepalived_lvs_status.log
Mon Oct 15 10:40:53 CST 2018:---------- health check start ...---------
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.1 ...
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.101 ...
Mon Oct 15 10:40:53 CST 2018:ping 192.168.56.102 ...
Mon Oct 15 10:40:53 CST 2018:fail ping num:0
Mon Oct 15 10:40:53 CST 2018:haproxy is not running, restart it
Mon Oct 15 10:40:53 CST 2018:check ok,exit 0

notify_master功能验证
haproxy1 节点上关闭 keepalived

# systemctl stop keepalived

haproxy2 节点上查看日志

# cat /var/log/keepalived_lvs_master.log 
Mon Oct 15 10:45:53 CST 2018:keepalived to master

notify_backup功能验证
haproxy1 节点再启动 keepalived,由于是不抢占模式，所以haproxy1起来后就是backup模式。

# systemctl start keepalived
# cat /var/log/keepalived_lvs_backup.log 
Mon Oct 15 11:15:46 CST 2018:keepalived to backup

此时，应用只需要连接 keepalived 的 vip 即可。

参考：
https://www.cnblogs.com/clsn/p/8052649.html
http://www.keepalived.org/