使用ZooKeeper协调多台Web Server的定时任务处理(方案2)

承接上个博文, 这次是方案2的实现, 本方案的特点:
1. 该方案能很好地从几台服务器中选出一个Master机器, 不仅仅可以用于定时任务场景, 还可以用在其他场景下.
2. 该方案能实现Master节点的自动 failover, 经我测试 failover 过程稍长, 接近1分钟.
综上所述, 本方案是推荐方案.

==============================
Curator 中 LeaderSelector 相关
==============================
会用到四个相关类, 分别是:
LeaderSelector - 选举Leader的底层类.
LeaderSelectorListener 接口 - 在本接口中, 引入了takeLeadership()虚拟函数, 可以用来监听获得领导权.
LeaderSelectorListenerAdapter - 官方提供一个LeaderSelectorListener实现类, 我们可以在这个标准的实现上增加业务逻辑, 一般只需要实现takeLeadership()方法即可.
CancelLeadershipException - 取消Leader权异常.

这几个接口和类的关系是:
ConnectionStateListener 接口(引入了 stateChanged 虚函数)
        |
        |
        v
LeaderSelectorListener 接口(引入了 takeLeadership 虚函数)
        |
        |
        v
LeaderSelectorListenerAdapter 类(实现了 stateChanged() 方法), 实现了一个标准的stateChanged()方法, 即当zk连接异常,
        |                      直接抛出 CancelLeadershipException, 该异常将触发leaderSelector.interruptLeadership, 即中断领导权.
        |
        |
        v
LeaderSelectorController 自定义类( 主要用来实现 takeLeadership 方法)

重要的方法说明:

leaderSelector.autoRequeue()
调用该方法, 可以确保相关zk client在释放领导权后能参与选举.

leaderSelector.start()
让zk 客户端立即参与选举

leaderSelector.close()
放弃竞选参与.

leaderSelector.hasLeadership()
检查是否是Leader, 返回值为boolean, 该函数调用会立即返回.

leaderSelectorListener.takeLeadership()
获取领导权被触发的函数, 可以在其中编写业务逻辑, 需要注意的是, 该函数一旦结束, 将自动释放领导权. 所以如果要hold住领导权的话, 该函数中要加一个死循环.

client.getConnectionStateListenable().addListener()
为zk 客户端增加一个监听器, 用来监听连接状态, 共有三种状态: RECONNECT/LOST/SUSPEND
当连接状态为 ConnectionState.LOST 时, 写代码强制客户端重连, 以便该客户端能继续参与Leader选举.
当连接状态为 ConnectionState.SUSPEND 时, 我们一般不用处理, 输出log即可.

=============================
环境准备
=============================
在 VM (192.168.1.11) 上启动一个 zookeeper 容器
docker run -d --name myzookeeper --net host zookeeper:latest

在Windows开发机上, 使用 zkCli.cmd 应该能连上虚机中的 zk server.
zkCli.cmd -server 192.168.1.11:2181

=============================
SpringBoot 服务程序
=============================
增加下面三个依赖项, 引入 actuator 仅仅是为了不写任何代码就能展现一个web UI.

<dependency>
    <groupId>org.apache.curator</groupId>
    <artifactId>curator-recipes</artifactId>
    <version>2.12.0</version>
</dependency>
<dependency>
    <!--org.apache.curator 依赖 slf4j -->
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-simple</artifactId>
    <version>1.7.7</version>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
 

Java 代码:

package com.example.demo;

import java.io.Closeable;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.leader.LeaderSelector;
import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

/*
 * 主程序类
 * */
@EnableScheduling
@SpringBootApplication
public class ZkServiceApplication {
    public static void main(String[] args) throws Exception {
        SpringApplication.run(ZkServiceApplication.class, args);

        //开始节点选举过程
        ZkLeaderSelectorController leaderController = new ZkLeaderSelectorController();
        leaderController.start();
        Thread.sleep(Long.MAX_VALUE);
        leaderController.close();
    }
}

/*
 * 常量工具类
 */
class ZkConst {
    public static final String SERVICE_NAME = "ServiceA";
    public static final String LATCH_NAME = "node";
    public static final String LEADER_NAME = "master";
    public static final String SERVICE_SERVER = "Server1:8080";
    public static final String ZK_URL = "localhost:2181";

    public static String getZkLatchPath() {
        return String.format("/%s/%s", SERVICE_NAME, LATCH_NAME);
    }

    public static String getZkLeaderPath() {
        return String.format("/%s/%s", SERVICE_NAME, LEADER_NAME);
    }

}

/*
 * Zk Connection 监听器 如果 zk client 长连接断开后, 需要重连以保证该客户端仍能参与 Leader 选举.
 * 对于定时任务级的Leader选举, 这个监听器并不重要. 对于服务器级别的Leader选举, 这个监听器很重要.
 */
class ZkConnectionStateListener implements ConnectionStateListener {
    private static final Logger log = LoggerFactory.getLogger(ZkConnectionStateListener.class);

    @Override
    public void stateChanged(CuratorFramework client, ConnectionState newState) {
        log.debug("Zk connection change to " + newState);
        if (ConnectionState.CONNECTED != newState) {
            while (true) {
                try {
                    log.error("Disconnected to the Zk server. Try to reconnect Zk server");
                    client.getZookeeperClient().blockUntilConnectedOrTimedOut();
                    log.info("Succeed to reconnect Zk server");
                } catch (InterruptedException e) {
                    log.error(e.getMessage(), e);
                }
            }
        }
    }

}


/*
 * Zk 工具类
 */
class ZkHelper {
    private static final Logger log = LoggerFactory.getLogger(ZkHelper.class);

    public static CuratorFramework getClient(boolean autoStart) {
        RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
        CuratorFramework client = CuratorFrameworkFactory.newClient(ZkConst.ZK_URL, retryPolicy);
        if (client.getState() != CuratorFrameworkState.STARTED && autoStart) {
            client.start();
        }
        return client;
    }

    public static boolean isMasterNode(CuratorFramework client) {
        boolean result = false;
        String nodeValue = "(nodeValue)";
        String path = ZkConst.getZkLeaderPath();
        Stat stat;
        try {
            stat = client.checkExists().forPath(path);
            if (stat != null) {
                nodeValue = new String(client.getData().forPath(path));
            }
            result = nodeValue.equals(ZkConst.SERVICE_SERVER);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return result;
    }

    public static boolean isMasterNode() {
        CuratorFramework client = getClient(false);
        // client.getConnectionStateListenable().addListener(new
        // ZkConnectionStateListener());
        client.start();
        boolean result = isMasterNode(client);
        client.close();
        return result;
    }

    public static void setMasterName(CuratorFramework client, boolean unknownMasterName) {
        String serverName;
        if (unknownMasterName) {
            serverName = "Master In electing";
        } else {
            serverName = ZkConst.SERVICE_SERVER;
        }
        String path = ZkConst.getZkLeaderPath();
        Stat stat;
        try {
            stat = client.checkExists().forPath(path);
            if (stat != null) {
                client.setData().forPath(path, serverName.getBytes());
            } else {
                client.create().forPath(path, serverName.getBytes());
            }

        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
    }

    public static void setMasterName(boolean unknownMasterName) {
        CuratorFramework client = getClient(false);
        client.getConnectionStateListenable().addListener(new ZkConnectionStateListener());
        client.start();
        setMasterName(client, unknownMasterName);
        client.close();
    }

}


/*
 * Master选举的控制类
 */
class ZkLeaderSelectorController extends LeaderSelectorListenerAdapter implements Closeable {
    private static final Logger log = LoggerFactory.getLogger(LeaderSelectorListenerAdapter.class);

    private LeaderSelector leaderSelector;
    private CuratorFramework client;
    String path;

    public ZkLeaderSelectorController() {
        client = ZkHelper.getClient(false);
        path = ZkConst.getZkLatchPath();
        leaderSelector = new LeaderSelector(client, path, this);
        leaderSelector.autoRequeue(); // 自动重新排队
    }

    /*
     * 获得Leader之后的处理过程
     */
    @Override
    public void takeLeadership(CuratorFramework client) throws Exception {
        log.info(String.format("The server (%s) become as master", ZkConst.SERVICE_SERVER));
        boolean unknownMasterName = false;
        ZkHelper.setMasterName(client, unknownMasterName);

        // 永远 hold 住领导权
        Thread.sleep(Long.MAX_VALUE);
        log.info(String.format("The server (%s) become as non-master", ZkConst.SERVICE_SERVER));
    }

    @Override
    public void stateChanged(CuratorFramework client, ConnectionState newState) {
        if ((newState == ConnectionState.SUSPENDED) || (newState == ConnectionState.LOST)) {
            log.error("Disconnected to the Zk server. Try to release leadership ");
        }

        super.stateChanged(client, newState);

        if ((newState == ConnectionState.SUSPENDED) || (newState == ConnectionState.LOST)) {
            boolean unknownMasterName = true;
            ZkHelper.setMasterName(client, unknownMasterName);
        }
    }

    /*
     * 开始竞争leader
     */
    public void start() {
        client.start();
        leaderSelector.start();
    }

    @Override
    public void close() throws IOException {
        leaderSelector.close();
        client.close();
    }
}

/*
 * 定时任务控制器类
 */
class ZkTaskController {
    private static final Logger log = LoggerFactory.getLogger(ZkTaskController.class);
    private String taskName;

    public ZkTaskController(String taskName) {
        this.taskName = taskName;
    }

    public void runTask(Runnable action) {
        if (ZkHelper.isMasterNode()) {
            log.info(String.format("The task %s will run on this master server", taskName));
            action.run();
        } else {
            log.info(String.format("The task %s will not run on this non-master server", taskName));
        }
    }
}

/*
 * 定时任务类
 */
@Component
class MyTasks {

    /**
     * 一个定时任务 reportCurrentTimeTask 方法 (每分钟运行)
     */
    @Scheduled(cron = "0 * * * * *")
    public void reportCurrentTimeTask() {
        ZkTaskController zkTaskController = new ZkTaskController("reportCurrentTime");
        zkTaskController.runTask(new ReportCurrentTimeTaskInternal());
    }

    /**
     * 定时任务 reportCurrentTimeTask 真正执行的内容
     */
    class ReportCurrentTimeTaskInternal implements Runnable {
        private final Logger log = LoggerFactory.getLogger(ReportCurrentTimeTaskInternal.class);
        private final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");

        @Override
        public void run() {
            log.info(String.format("The server (%s) is now %s", ZkConst.SERVICE_SERVER, dateFormat.format(new Date())));
        }
    }

}