部署环境: CentOS7.5
192.168.94.11 (master)
192.168.94.22 (slave0)
192.168.94.33 (slave1)
192.168.94.44 (slave2)
所有节点都需要安装redis+sentinel
[root@master ~]# tar xf redis-4.0.11.tar.gz -C /usr/local/src/ [root@master ~]# cd /usr/local/src/redis-4.0.11/ [root@master redis-4.0.11]# make && make PREFIX=/usr/local/redis install [root@master redis-4.0.11]# cd /usr/local/redis/ [root@master redis]# mkdir run conf log dir [root@master redis]# ln -s /usr/local/redis/bin/* /usr/local/bin/
所有节点修改redis和sentinel配置文件
[root@master redis]# cp /usr/local/src/redis-4.0.11/redis.conf conf/ bind 0.0.0.0 protected-mode no port 6379 tcp-backlog 511 timeout 0 tcp-keepalive 300 daemonize yes supervised no pidfile /usr/local/redis/run/redis_6379.pid loglevel notice logfile "/usr/local/redis/log/redis_6379.log" databases 16 always-show-logo yes save 900 1 save 300 10 save 60 10000 stop-writes-on-bgsave-error yes rdbcompression yes rdbchecksum yes dbfilename dump.rdb dir /usr/local/redis/dir slave-serve-stale-data yes slave-read-only yes repl-diskless-sync no repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no slave-priority 100 lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no slave-lazy-flush no appendonly yes appendfilename "appendonly.aof" appendfsync everysec no-appendfsync-on-rewrite no auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb aof-load-truncated yes aof-use-rdb-preamble no lua-time-limit 5000 slowlog-log-slower-than 10000 slowlog-max-len 128 latency-monitor-threshold 0 notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 list-max-ziplist-size -2 list-compress-depth 0 set-max-intset-entries 512 zset-max-ziplist-entries 128 zset-max-ziplist-value 64 hll-sparse-max-bytes 3000 activerehashing yes client-output-buffer-limit normal 0 0 0 client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 hz 10 aof-rewrite-incremental-fsync yes # slave上要添加一条: slaveof 192.168.94.11 6379 [root@master redis]# cp /usr/local/src/redis-4.0.11/sentinel.conf conf/ [root@master ~]# grep -Ev "^#|^$" /usr/local/redis/conf/sentinel.conf port 26379 pidfile "/usr/local/redis/run/redis-sentinel.pid" dir "/usr/local/redis/dir/" daemonize yes protected-mode no logfile "/usr/local/redis/log/redis-sentinel.log" sentinel monitor mymaster 192.168.94.11 6379 2 sentinel down-after-milliseconds mymaster 10000 sentinel parallel-syncs mymaster 1 sentinel failover-timeout mymaster 60000
添加redis到系统服务
[root@master redis]# cp /usr/local/src/redis-4.0.11/utils/redis_init_script /etc/init.d/redis [root@master ~]# vim /etc/init.d/redis #!/bin/sh # # Simple Redis init.d script conceived to work on Linux systems # as it does use of the /proc filesystem. ### BEGIN INIT INFO # Provides: redis_6379 # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Redis data structure server # Description: Redis data structure server. See https://redis.io ### END INIT INFO REDISPORT=6379 EXEC=/usr/local/bin/redis-server CLIEXEC=/usr/local/bin/redis-cli PIDFILE=/usr/local/redis/run/redis_${REDISPORT}.pid CONF="/usr/local/redis/conf/redis.conf" case "$1" in start) if [ -f $PIDFILE ] then echo "$PIDFILE exists, process is already running or crashed" else echo "Starting Redis server..." $EXEC $CONF fi ;; stop) if [ ! -f $PIDFILE ] then echo "$PIDFILE does not exist, process is not running" else PID=$(cat $PIDFILE) echo "Stopping ..." $CLIEXEC -p $REDISPORT shutdown while [ -x /proc/${PID} ] do echo "Waiting for Redis to shutdown ..." sleep 1 done echo "Redis stopped" fi ;; *) echo "Please use start or stop as first argument" ;; esac [root@master redis]# chkconfig --add redis [root@master redis]# systemctl daemon-reload
所有节点启动redis、sentinel服务器
[root@master redis]# systemctl start redis [root@master redis]# systemctl status redis ● redis.service - LSB: Redis data structure server Loaded: loaded (/etc/rc.d/init.d/redis; bad; vendor preset: disabled) Active: active (running) since 三 2018-09-05 17:31:43 CST; 3s ago Docs: man:systemd-sysv-generator(8) Process: 5933 ExecStart=/etc/rc.d/init.d/redis start (code=exited, status=0/SUCCESS) CGroup: /system.slice/redis.service └─5935 /usr/local/bin/redis-server 0.0.0.0:6379 9月 05 17:31:43 master systemd[1]: Starting LSB: Redis data structure server... 9月 05 17:31:43 master redis[5933]: Starting Redis server... 9月 05 17:31:43 master systemd[1]: Started LSB: Redis data structure server. [root@master redis]# redis-sentinel /usr/local/redis/conf/sentinel.conf [root@master redis]# netstat -antup|grep 26379 tcp 0 0 0.0.0.0:26379 0.0.0.0:* LISTEN 23409/redis-sentine tcp6 0 0 :::26379 :::* LISTEN 23409/redis-sentine
查看redis和sentinel信息
# 查看节点redis的主从关系 [root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info|grep role role:master [root@master redis]# redis-cli -h 192.168.94.22 -p 6379 info|grep role role:slave [root@master redis]# redis-cli -h 192.168.94.33 -p 6379 info|grep role role:slave [root@master redis]# redis-cli -h 192.168.94.44 -p 6379 info|grep role role:slave # 查看master节点信息 [root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info Replication # Replication role:master connected_slaves:3 slave0:ip=192.168.94.33,port=6379,state=online,offset=23362,lag=1 slave1:ip=192.168.94.44,port=6379,state=online,offset=23362,lag=1 slave2:ip=192.168.94.22,port=6379,state=online,offset=23362,lag=1 master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e master_replid2:0000000000000000000000000000000000000000 master_repl_offset:23503 second_repl_offset:-1 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:1 repl_backlog_histlen:23503 # 查看master的sentinel配置文件 , 末尾被写入如下内容 [root@master redis]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf port 26379 dir "/usr/local/redis/dir" sentinel myid 2b92a48338bcd05968782fb3e4af432ad388e1c6 sentinel deny-scripts-reconfig yes sentinel monitor mymaster 192.168.94.11 6379 2 sentinel down-after-milliseconds mymaster 10000 sentinel failover-timeout mymaster 60000 pidfile "/usr/local/redis/run/redis_sentinel.pid" logfile "/usr/local/redis/log/redis_sentinel.log" daemonize yes sentinel config-epoch mymaster 0 sentinel leader-epoch mymaster 0 sentinel known-slave mymaster 192.168.94.33 6379 sentinel known-slave mymaster 192.168.94.22 6379 sentinel known-slave mymaster 192.168.94.44 6379 sentinel known-sentinel mymaster 192.168.94.33 26379 cd7c8822acb184eb1af4f96710fc93f5073cfc65 sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181 sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06 sentinel current-epoch 0 # 查看slave0的节点信息 [root@slave0 ~]# redis-cli -h 192.168.94.33 -p 6379 info Replication # Replication role:slave master_host:192.168.94.11 master_port:6379 master_link_status:up master_last_io_seconds_ago:0 master_sync_in_progress:0 slave_repl_offset:142968 slave_priority:100 slave_read_only:1 connected_slaves:0 master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e master_replid2:0000000000000000000000000000000000000000 master_repl_offset:142968 second_repl_offset:-1 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:1 repl_backlog_histlen:142968 # 查看slave的sentinel配置文件 , 在末尾处被写入如下内容 [root@slave0 ~]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf port 26379 dir "/usr/local/redis/dir" sentinel myid cd7c8822acb184eb1af4f96710fc93f5073cfc65 sentinel deny-scripts-reconfig yes sentinel monitor mymaster 192.168.94.11 6379 3 sentinel down-after-milliseconds mymaster 10000 sentinel failover-timeout mymaster 60000 pidfile "/usr/local/redis/run/redis_sentinel.pid" logfile "/usr/local/redis/log/redis_sentinel.log" daemonize yes sentinel config-epoch mymaster 0 sentinel leader-epoch mymaster 0 sentinel known-slave mymaster 192.168.94.22 6379 sentinel known-slave mymaster 192.168.94.33 6379 sentinel known-slave mymaster 192.168.94.44 6379 sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06 sentinel known-sentinel mymaster 192.168.94.11 26379 2b92a48338bcd05968782fb3e4af432ad388e1c6 sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181 sentinel current-epoch 0 # 其他slave节点也是一样的
测试客户端写入数据
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 192.168.94.11:6379> set mywife limingming OK 192.168.94.11:6379> get mywife "limingming" 192.168.94.11:6379> exit # 客户端连接任意slave节点get上面的数据 [root@master redis]# redis-cli -h 192.168.94.22 -p 6379 192.168.94.22:6379> get mywife "limingming" 192.168.94.22:6379> set limingming xiaoxiannv (error) READONLY You can't write against a read only slave. # 测试可见 , master节点可以写入 , slave节点默认是只读 ,不能写入 到这就已经实现主从复制 , 读写分离了
通过sentinel实现主从切换 , sentinel需要做高可用防止单点
# 关闭任意slave节点 , 查看状态 [root@master redis]# redis-cli -h 192.168.94.22 -p 6379 192.168.94.22:6379> get mywife "limingming" 192.168.94.22:6379> SHUTDOWN not connected> exit # 查看剩余节点信息 [root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication # Replication role:master connected_slaves:2 # 由3变成2了 , 减少了一台slave slave0:ip=192.168.94.33,port=6379,state=online,offset=410139,lag=1 slave1:ip=192.168.94.44,port=6379,state=online,offset=410280,lag=0 master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e master_replid2:0000000000000000000000000000000000000000 master_repl_offset:410280 second_repl_offset:-1 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:1 repl_backlog_histlen:410280 # 在任意节点查看sentinel日志 , 发现slave已经是sdown状态 [root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 # 然后重启上面被关闭的节点 , 所有节点的sentinel都可以检测到slave已处于可用状态，此时再来查看节点的主从信息 [root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication # Replication role:master connected_slaves:3 slave0:ip=192.168.94.33,port=6379,state=online,offset=5900717,lag=1 slave1:ip=192.168.94.44,port=6379,state=online,offset=5900858,lag=1 slave2:ip=192.168.94.22,port=6379,state=online,offset=5900858,lag=1 master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e master_replid2:0000000000000000000000000000000000000000 master_repl_offset:5900858 second_repl_offset:-1 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:4852283 repl_backlog_histlen:1048576 # 日志已经显示slave进入-sdown状态 [root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 23409:X 06 Sep 00:53:01.203 * +reboot slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 23409:X 06 Sep 00:53:01.261 # -sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
关掉master节点 , 观察slave节点
[root@master ~]# redis-cli -h 192.168.94.11 -p 6379 192.168.94.11:6379> SHUTDOWN # 在任意节点查看sentinel日志 , 发现master已经是+sdown的状态 [root@slave2 ~]# tailf /usr/local/redis/log/redis_sentinel.log 1279:X 06 Sep 11:06:30.806 * +slave-reconf-done slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379 1279:X 06 Sep 11:06:30.869 * +slave-reconf-sent slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 1279:X 06 Sep 11:06:31.821 * +slave-reconf-inprog slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 1279:X 06 Sep 11:06:31.821 * +slave-reconf-done slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379 1279:X 06 Sep 11:06:31.910 # +failover-end master mymaster 192.168.94.11 6379 1279:X 06 Sep 11:06:31.911 # +switch-master mymaster 192.168.94.11 6379 192.168.94.44 6379 1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.44 6379 1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.44 6379 1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379 1279:X 06 Sep 11:06:41.995 # +sdown slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379 [root@slave2 ~]# redis-cli -h 192.168.94.22 -p 6379 info |grep role role:slave [root@slave2 ~]# redis-cli -h 192.168.94.33 -p 6379 info |grep role role:slave [root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info |grep role role:master [root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication # Replication role:master connected_slaves:2 slave0:ip=192.168.94.33,port=6379,state=online,offset=224153,lag=0 slave1:ip=192.168.94.22,port=6379,state=online,offset=223730,lag=1 master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289 master_replid2:086f725685de634b2b3efd546d150a370e461d07 master_repl_offset:224153 second_repl_offset:62861 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:1 repl_backlog_histlen:224153 # 可以看出 , 当master节点(192.168.94.11)的redis关闭后 , slave节点(192.168.94.44)变成了新的master # 在这个新的master上进行写的操作是可以执行的 [root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 192.168.94.44:6379> set mywife limingming OK 192.168.94.44:6379> get mywife "limingming" # 重启192.168.94.11节点的redis , 等待所有节点的sentinel都检测到后 ,查看所有节点的主从信息 , 现有的master节点是不会被强占去的 , 而原来192.168.94.11节点的角色由master变成了slave [root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info | grep role role:master [root@slave1 ~]# redis-cli -h 192.168.94.22 -p 6379 info | grep role role:slave [root@slave1 ~]# redis-cli -h 192.168.94.33 -p 6379 info | grep role role:slave [root@slave1 ~]# redis-cli -h 192.168.94.11 -p 6379 info | grep role role:slave [root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication # Replication role:master connected_slaves:3 slave0:ip=192.168.94.33,port=6379,state=online,offset=406595,lag=0 slave1:ip=192.168.94.22,port=6379,state=online,offset=406595,lag=0 slave2:ip=192.168.94.11,port=6379,state=online,offset=406595,lag=0 master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289 master_replid2:086f725685de634b2b3efd546d150a370e461d07 master_repl_offset:406595 second_repl_offset:62861 repl_backlog_active:1 repl_backlog_size:1048576 repl_backlog_first_byte_offset:1 repl_backlog_histlen:406595 # 所有slave将同步新的master上的信息
主从故障切换已经实现了