Redis哨兵模式（sentinel）部署记录（主从复制、读写分离、主从切换）

部署环境: CentOS7.5 

192.168.94.11 (master)

192.168.94.22 (slave0)

192.168.94.33 (slave1)

192.168.94.44 (slave2)

所有节点都需要安装redis+sentinel

[root@master ~]# tar xf redis-4.0.11.tar.gz -C  /usr/local/src/
[root@master ~]# cd /usr/local/src/redis-4.0.11/
[root@master redis-4.0.11]# make && make PREFIX=/usr/local/redis install
[root@master redis-4.0.11]# cd /usr/local/redis/
[root@master redis]# mkdir run conf log dir
[root@master redis]# ln -s /usr/local/redis/bin/* /usr/local/bin/

所有节点修改redis和sentinel配置文件

[root@master redis]# cp /usr/local/src/redis-4.0.11/redis.conf conf/
bind 0.0.0.0
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /usr/local/redis/run/redis_6379.pid
loglevel notice
logfile "/usr/local/redis/log/redis_6379.log"
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/redis/dir
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
# slave上要添加一条: slaveof 192.168.94.11 6379

[root@master redis]# cp /usr/local/src/redis-4.0.11/sentinel.conf conf/
[root@master ~]# grep -Ev "^#|^$" /usr/local/redis/conf/sentinel.conf 
port 26379
pidfile "/usr/local/redis/run/redis-sentinel.pid"
dir "/usr/local/redis/dir/"
daemonize yes
protected-mode no
logfile "/usr/local/redis/log/redis-sentinel.log"
sentinel monitor mymaster 192.168.94.11 6379 2 
sentinel down-after-milliseconds mymaster 10000 
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 60000 

添加redis到系统服务

[root@master redis]# cp /usr/local/src/redis-4.0.11/utils/redis_init_script /etc/init.d/redis
[root@master ~]# vim /etc/init.d/redis 
#!/bin/sh
#
# Simple Redis init.d script conceived to work on Linux systems
# as it does use of the /proc filesystem.

### BEGIN INIT INFO
# Provides:     redis_6379
# Default-Start:        2 3 4 5
# Default-Stop:         0 1 6
# Short-Description:    Redis data structure server
# Description:          Redis data structure server. See https://redis.io
### END INIT INFO

REDISPORT=6379
EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/usr/local/redis/run/redis_${REDISPORT}.pid
CONF="/usr/local/redis/conf/redis.conf"
case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
                echo "$PIDFILE exists, process is already running or crashed"
        else
                echo "Starting Redis server..."
                $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
                echo "$PIDFILE does not exist, process is not running"
        else
                PID=$(cat $PIDFILE)
                echo "Stopping ..."
                $CLIEXEC -p $REDISPORT shutdown
                while [ -x /proc/${PID} ]
                do
                    echo "Waiting for Redis to shutdown ..."
                    sleep 1
                done
                echo "Redis stopped"
        fi
        ;;
    *)
        echo "Please use start or stop as first argument"
        ;;
esac

[root@master redis]# chkconfig --add redis
[root@master redis]# systemctl daemon-reload

所有节点启动redis、sentinel服务器

[root@master redis]# systemctl start redis
[root@master redis]# systemctl status redis
● redis.service - LSB: Redis data structure server
   Loaded: loaded (/etc/rc.d/init.d/redis; bad; vendor preset: disabled)
   Active: active (running) since 三 2018-09-05 17:31:43 CST; 3s ago
     Docs: man:systemd-sysv-generator(8)
  Process: 5933 ExecStart=/etc/rc.d/init.d/redis start (code=exited, status=0/SUCCESS)
   CGroup: /system.slice/redis.service
           └─5935 /usr/local/bin/redis-server 0.0.0.0:6379

9月 05 17:31:43 master systemd[1]: Starting LSB: Redis data structure server...
9月 05 17:31:43 master redis[5933]: Starting Redis server...
9月 05 17:31:43 master systemd[1]: Started LSB: Redis data structure server.

[root@master redis]# redis-sentinel /usr/local/redis/conf/sentinel.conf
[root@master redis]# netstat -antup|grep 26379
tcp        0      0 0.0.0.0:26379           0.0.0.0:*               LISTEN      23409/redis-sentine 
tcp6       0      0 :::26379                :::*                    LISTEN      23409/redis-sentine 

查看redis和sentinel信息

# 查看节点redis的主从关系
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info|grep role
role:master
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379 info|grep role
role:slave
[root@master redis]# redis-cli -h 192.168.94.33 -p 6379 info|grep role
role:slave
[root@master redis]# redis-cli -h 192.168.94.44 -p 6379 info|grep role
role:slave

# 查看master节点信息
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info Replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=23362,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=23362,lag=1
slave2:ip=192.168.94.22,port=6379,state=online,offset=23362,lag=1
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:23503
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:23503

# 查看master的sentinel配置文件 , 末尾被写入如下内容
[root@master redis]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf 
port 26379
dir "/usr/local/redis/dir"
sentinel myid 2b92a48338bcd05968782fb3e4af432ad388e1c6
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 192.168.94.11 6379 2
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 60000
pidfile "/usr/local/redis/run/redis_sentinel.pid"
logfile "/usr/local/redis/log/redis_sentinel.log"
daemonize yes
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel known-slave mymaster 192.168.94.33 6379
sentinel known-slave mymaster 192.168.94.22 6379
sentinel known-slave mymaster 192.168.94.44 6379
sentinel known-sentinel mymaster 192.168.94.33 26379 cd7c8822acb184eb1af4f96710fc93f5073cfc65
sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181
sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06
sentinel current-epoch 0

# 查看slave0的节点信息
[root@slave0 ~]# redis-cli -h 192.168.94.33 -p 6379 info Replication
# Replication
role:slave
master_host:192.168.94.11
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:142968
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:142968
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:142968

# 查看slave的sentinel配置文件 , 在末尾处被写入如下内容
[root@slave0 ~]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf 
port 26379
dir "/usr/local/redis/dir"
sentinel myid cd7c8822acb184eb1af4f96710fc93f5073cfc65
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 192.168.94.11 6379 3
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 60000
pidfile "/usr/local/redis/run/redis_sentinel.pid"
logfile "/usr/local/redis/log/redis_sentinel.log"
daemonize yes
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel known-slave mymaster 192.168.94.22 6379
sentinel known-slave mymaster 192.168.94.33 6379
sentinel known-slave mymaster 192.168.94.44 6379
sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06
sentinel known-sentinel mymaster 192.168.94.11 26379 2b92a48338bcd05968782fb3e4af432ad388e1c6
sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181
sentinel current-epoch 0
# 其他slave节点也是一样的

测试客户端写入数据

[root@master redis]# redis-cli -h 192.168.94.11 -p 6379
192.168.94.11:6379> set mywife limingming
OK
192.168.94.11:6379> get mywife
"limingming"
192.168.94.11:6379> exit
# 客户端连接任意slave节点get上面的数据
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379
192.168.94.22:6379> get mywife
"limingming"
192.168.94.22:6379> set limingming xiaoxiannv
(error) READONLY You can't write against a read only slave.

# 测试可见 , master节点可以写入 , slave节点默认是只读 ,不能写入 
到这就已经实现主从复制 , 读写分离了 

通过sentinel实现主从切换 , sentinel需要做高可用防止单点

# 关闭任意slave节点 , 查看状态
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379
192.168.94.22:6379> get mywife
"limingming"
192.168.94.22:6379> SHUTDOWN
not connected> exit

# 查看剩余节点信息
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication
# Replication
role:master
connected_slaves:2     # 由3变成2了 , 减少了一台slave 
slave0:ip=192.168.94.33,port=6379,state=online,offset=410139,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=410280,lag=0
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:410280
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:410280

# 在任意节点查看sentinel日志 ,  发现slave已经是sdown状态
[root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 
23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379

# 然后重启上面被关闭的节点 ,  所有节点的sentinel都可以检测到slave已处于可用状态，此时再来查看节点的主从信息 
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=5900717,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=5900858,lag=1
slave2:ip=192.168.94.22,port=6379,state=online,offset=5900858,lag=1
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:5900858
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:4852283
repl_backlog_histlen:1048576

# 日志已经显示slave进入-sdown状态
[root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 
23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 06 Sep 00:53:01.203 * +reboot slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 06 Sep 00:53:01.261 # -sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379

关掉master节点 , 观察slave节点

[root@master ~]# redis-cli -h 192.168.94.11 -p 6379
192.168.94.11:6379> SHUTDOWN

# 在任意节点查看sentinel日志 , 发现master已经是+sdown的状态
[root@slave2 ~]# tailf /usr/local/redis/log/redis_sentinel.log 
1279:X 06 Sep 11:06:30.806 * +slave-reconf-done slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:30.869 * +slave-reconf-sent slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.821 * +slave-reconf-inprog slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.821 * +slave-reconf-done slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.910 # +failover-end master mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.911 # +switch-master mymaster 192.168.94.11 6379 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:41.995 # +sdown slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379

[root@slave2 ~]# redis-cli -h 192.168.94.22 -p 6379 info |grep role
role:slave
[root@slave2 ~]# redis-cli -h 192.168.94.33 -p 6379 info |grep role
role:slave
[root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info |grep role
role:master

[root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication
# Replication
role:master
connected_slaves:2
slave0:ip=192.168.94.33,port=6379,state=online,offset=224153,lag=0
slave1:ip=192.168.94.22,port=6379,state=online,offset=223730,lag=1
master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289
master_replid2:086f725685de634b2b3efd546d150a370e461d07
master_repl_offset:224153
second_repl_offset:62861
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:224153

# 可以看出 , 当master节点(192.168.94.11)的redis关闭后 , slave节点(192.168.94.44)变成了新的master

# 在这个新的master上进行写的操作是可以执行的
[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 
192.168.94.44:6379> set mywife limingming
OK
192.168.94.44:6379> get mywife
"limingming"

# 重启192.168.94.11节点的redis , 等待所有节点的sentinel都检测到后 ,查看所有节点的主从信息 , 现有的master节点是不会被强占去的 , 而原来192.168.94.11节点的角色由master变成了slave
[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info | grep role
role:master
[root@slave1 ~]# redis-cli -h 192.168.94.22 -p 6379 info | grep role
role:slave
[root@slave1 ~]# redis-cli -h 192.168.94.33 -p 6379 info | grep role
role:slave
[root@slave1 ~]# redis-cli -h 192.168.94.11 -p 6379 info | grep role
role:slave

[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=406595,lag=0
slave1:ip=192.168.94.22,port=6379,state=online,offset=406595,lag=0
slave2:ip=192.168.94.11,port=6379,state=online,offset=406595,lag=0
master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289
master_replid2:086f725685de634b2b3efd546d150a370e461d07
master_repl_offset:406595
second_repl_offset:62861
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:406595
# 所有slave将同步新的master上的信息 
主从故障切换已经实现了