Java网络编程学习A轮_08_NIO的Reactor模型

参考资料：
了解 Java NIO 的 Reactor 模型，大神 Doug Lea 的 PPT Scalable IO in Java 必看：http://gee.cs.oswego.edu/dl/cpjslides/nio.pdf

示例代码：
https://github.com/gordonklg/study，socket module

A. 单线程版

Reactor 相当于一个中央事件收集分发器。一方面，Reactor 通过 Selector 可以收到已经准备完毕的事件通知，另一方面，Reactor 将事件发送给对应的 Handler 处理。

对于 NIO 服务端，建立连接与数据传输是通过不同类型的 Channel 处理的。ServerSocketChannel 用来处理连接建立请求，其 accept 方法创建出的 SocketChannel 用来处理与客户端的数据传输。多数情况下，服务端会有一个 ServerSocketChannel 以及数量与已连接客户端总数一致的 SocketChannel。在单线程版 Reactor 模型中，所有的 Channel 都会注册到 Reactor 的 Selector 上，由 Reactor 的事件循环代码分发（dispatch）事件。

Reactor 将事件发送给对应的 Handler 处理，acceptor 可以看作一个特殊的 Handler，用于处理连接建立请求。而每个具体的连接（对应 SocketChannel）对应一个 Handler 实例，该 Handler 实例负责读取数据、解码、执行业务逻辑、编码以及发送数据给客户端。显然，Handler 是线程安全的。

nio.pdf 文件中包含了单线程版几乎所有的源码，自己只需要实现几个简单方法即可。我额外增加了一些调试日志用于观察系统运行情况。

gordon.study.socket.nio.reactor.singlethread.Reactor.java

public class Reactor implements Runnable {

    final Selector selector;

    final ServerSocketChannel serverSocket;

    public Reactor(int port) throws IOException {
        selector = Selector.open();
        serverSocket = ServerSocketChannel.open();
        serverSocket.socket().bind(new InetSocketAddress(port));
        serverSocket.configureBlocking(false);
        SelectionKey sk = serverSocket.register(selector, SelectionKey.OP_ACCEPT);
        sk.attach(new Acceptor());
    }

    @Override
    public void run() {
        try {
            while (!Thread.interrupted()) {
                selector.select();
                Set<SelectionKey> selected = selector.selectedKeys();
                printSelectedKeys(selected);
                Iterator<SelectionKey> it = selected.iterator();
                while (it.hasNext()) {
                    dispatch(it.next());
                }
                selected.clear();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void dispatch(SelectionKey key) {
        Runnable r = (Runnable) key.attachment();
        if (r != null) {
            r.run();
        }
    }

    private void printSelectedKeys(Set<SelectionKey> selected) {
        List<String> keyInfo = new ArrayList<>(selected.size());
        for (SelectionKey sk : selected) {
            String channelInfo = "";
            if (sk.channel() instanceof SocketChannel) {
                channelInfo = "SocketChannel port " + ((SocketChannel) sk.channel()).socket().getPort();
            } else {
                channelInfo = "ServerSocketChannel";
            }
            String readyOps = "";
            if ((sk.readyOps() & SelectionKey.OP_ACCEPT) > 0) {
                readyOps += "ACCEPT ";
            }
            if ((sk.readyOps() & SelectionKey.OP_CONNECT) > 0) {
                readyOps += "CONN ";
            }
            if ((sk.readyOps() & SelectionKey.OP_READ) > 0) {
                readyOps += "READ ";
            }
            if ((sk.readyOps() & SelectionKey.OP_WRITE) > 0) {
                readyOps += "WRITE ";
            }
            String interestOps = "";
            if ((sk.interestOps() & SelectionKey.OP_ACCEPT) > 0) {
                interestOps += "ACCEPT ";
            }
            if ((sk.interestOps() & SelectionKey.OP_CONNECT) > 0) {
                interestOps += "CONN ";
            }
            if ((sk.interestOps() & SelectionKey.OP_READ) > 0) {
                interestOps += "READ ";
            }
            if ((sk.interestOps() & SelectionKey.OP_WRITE) > 0) {
                interestOps += "WRITE ";
            }
            keyInfo.add(String.format("[%s, interestOps: %s, readyOps: %s]", channelInfo, interestOps, readyOps));
        }
        System.out.println(String.join(", ", keyInfo));
    }

    private class Acceptor implements Runnable {

        @Override
        public void run() {
            try {
                SocketChannel c = serverSocket.accept();
                if (c != null) {
                    new Handler(selector, c);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

gordon.study.socket.nio.reactor.singlethread.Handler.java

public final class Handler implements Runnable {

    private final SocketChannel socket;

    private final SelectionKey sk;

    private ByteBuffer input = ByteBuffer.allocate(1024);

    private ByteBuffer output = ByteBuffer.allocate(1024);

    private static final int READING = 0, SENDING = 1;

    private int state = READING;

    public Handler(Selector sel, SocketChannel c) throws IOException {
        socket = c;
        c.configureBlocking(false);
        sk = socket.register(sel, 0);
        sk.attach(this);
        sk.interestOps(SelectionKey.OP_READ);
    }

    @Override
    public void run() {
        try {
            if (state == READING) {
                read();
            } else if (state == SENDING) {
                send();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private boolean inputIsComplete() {
        if (input.position() >= 4) {
            int length = input.getInt();
            return input.position() >= length;
        }
        return false;
    }

    private boolean outputIsComplete() {
        return output.remaining() == 0;
    }

    private void process() {
        input.flip();
        byte[] bytes = new byte[input.getInt() - 4];
        input.get(bytes);
        String msg = new String(bytes);
        int remotePort = socket.socket().getPort();
        System.out.println(remotePort + " Processing ... " + msg);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
        System.out.println(remotePort + " Processed ... " + msg);
        output.put((byte) 'Y');
        output.flip();
    }

    private void read() throws IOException {
        socket.read(input);
        if (inputIsComplete()) {
            process();
            state = SENDING;
            sk.interestOps(SelectionKey.OP_WRITE);
        }
    }

    private void send() throws IOException {
        socket.write(output);
        if (outputIsComplete()) {
            sk.cancel();
        }
    }
}

gordon.study.socket.nio.reactor.singlethread.Main.java

public class Main {

    public static void main(String[] args) throws IOException {
        new Thread(new Reactor(8888)).start();
        for (int i = 0; i < 10; i++) {
            new Thread(new SocketClient()).start();
        }
    }

    private static class SocketClient implements Runnable {

        private String[] msgArray = { "ni hao", "hello", "chi le ma?", "你瞅啥？", "hi dude" };

        @Override
        public void run() {
            try (Socket socket = new Socket()) {
                socket.connect(new InetSocketAddress(8888));
                System.out.printf("  client (port %d) connected to server.
", socket.getLocalPort());
                DataInputStream dis = new DataInputStream(socket.getInputStream());
                DataOutputStream dos = new DataOutputStream(socket.getOutputStream());
                int pos = ThreadLocalRandom.current().nextInt(msgArray.length);
                sendMsg(msgArray[pos], dos);
                char result = (char) dis.read();
                System.out.printf("  client (port %d) get response from server: %s
", socket.getLocalPort(), result);
                dis.close();
                dos.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        private void sendMsg(String msg, DataOutputStream dos) throws Exception {
            byte[] bytes = msg.getBytes();
            int totalLength = 4 + bytes.length;
            dos.writeInt(totalLength);
            dos.write(bytes);
        }
    }
}

分析代码可以看出 Selector 是 Reactor 的核心，所有的 Channel 都会注册到 Selector 上。每个 Channel 对应一个 Handler 实例（Acceptor 视为特殊的 Handler），该 Handler 实例作为附件（attachment）附加在 SelectionKey 上。

这样，分发逻辑就十分简单了：当 Selector 选出 selectedKeys 时，遍历每个 key，拿出其附带的 Handler，执行其 run 方法即可。显然，Acceptor 与 Handler 实现 Runnable 接口的目的并不是为了多线程，只是为了有个共同的抽象（定义一个 AbstractHandler 替换 Runnable 的使用会更加容易理解一些）。

执行 main 方法，可以感受到单线程的效率低下（因为业务逻辑中 sleep 了一秒钟）。

有一个疑惑是关于 readyOps，从日志可以看出，当客户端发送了数据，SelectionKey 被选中时，readyOps 居然只包含 READ，而不包含 WRITE。在原来的理解中，这时 Channel 是可以向客户端发送数据的，所以 readyOps 应该包含 WRITE 才对啊？虽然可以写一段代码确认 readyOps 是不是受限于当前的 interestOps，但是意义不大，交给下一轮学习直接看源码吧。

B. 工作线程池版

一个最直接的优化思路就是将解码、业务处理和编码这些与 IO 无关的操作放到工作线程池中运行，以提高的 Reactor 的效率。

gordon.study.socket.nio.reactor.multithread.Handler.java

public final class Handler implements Runnable {

    private final SocketChannel socket;

    private final SelectionKey sk;

    private ByteBuffer input = ByteBuffer.allocate(1024);

    private ByteBuffer output = ByteBuffer.allocate(1024);

    private static Executor executor = Executors.newCachedThreadPool();
    
    private static final int READING = 0, SENDING = 1, PROCESSING = 2;

    private int state = READING;

    public Handler(Selector sel, SocketChannel c) throws IOException {
        socket = c;
        c.configureBlocking(false);
        sk = socket.register(sel, 0);
        sk.attach(this);
        sk.interestOps(SelectionKey.OP_READ);
    }

    @Override
    public void run() {
        try {
            if (state == READING) {
                read();
            } else if (state == SENDING) {
                send();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private boolean inputIsComplete() {
        if (input.position() >= 4) {
            int length = input.getInt();
            return input.position() >= length;
        }
        return false;
    }

    private boolean outputIsComplete() {
        return output.remaining() == 0;
    }

    private void process() {
        input.flip();
        byte[] bytes = new byte[input.getInt() - 4];
        input.get(bytes);
        String msg = new String(bytes);
        int remotePort = socket.socket().getPort();
        System.out.println(remotePort + " Processing ... " + msg);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
        System.out.println(remotePort + " Processed ... " + msg);
        output.put((byte) 'Y');
        output.flip();
    }

    private void read() throws IOException {
        socket.read(input);
        if (inputIsComplete()) {
            state = PROCESSING;
            executor.execute(new Processer());
        }
    }

    private void send() throws IOException {
        socket.write(output);
        if (outputIsComplete()) {
            sk.cancel();
        }
    }

    private void processAndHandOff() {
        process();
        state = SENDING;
        sk.interestOps(SelectionKey.OP_WRITE);
        sk.selector().wakeup(); // important!
    }

    private class Processer implements Runnable {
        public void run() {
            processAndHandOff();
        }
    }
}

关于代码，在我看来，Handler 依然是线程安全的，所以没必要使用 synchronize 关键字。

第85行调用 selector 的 wakeup 方法很重要，否则服务端不会将响应内容发送给客户端。表面上的原因显然是第84行的 set interest ops 操作（发生在某个工作线程中）没有对 Reactor 所在线程（即主线程）当前阻塞的方法 Selector.select() 生效，所以通过 wakeup 方法强行终止掉本次阻塞，以期待下次 select 方法能接收到该 Channel 的 WRITE 事件，使 dispatch 循环正常运行下去。深层原因（为什么不生效）留给下一轮看源码分析吧。

C. 多 Reactor 版

如 06 篇所分析，处理连接建立的 Reactor（Selector）与处理数据传输的 Reactor（Selector）需要分开。更进一步，为了利用多核的能力，处理数据传输的 Reactor 应该有多个。
下图中 mainReactor 用于处理连接，subReactor 用于处理数据传输。

gordon.study.socket.nio.reactor.multireactor.Reactor.java

public class Reactor implements Runnable {

    private Selector selector;

    private ServerSocketChannel serverSocket;

    public Reactor(int port) throws IOException {
        selector = Selector.open();
        serverSocket = ServerSocketChannel.open();
        serverSocket.socket().bind(new InetSocketAddress(port));
        serverSocket.configureBlocking(false);
        SelectionKey sk = serverSocket.register(selector, SelectionKey.OP_ACCEPT);
        sk.attach(new Acceptor());
    }

    public Reactor(Selector selector) throws IOException {
        this.selector = selector;
    }

    @Override
    public void run() {
        try {
            while (!Thread.interrupted()) {
                if(selector.select(100) == 0) {
                    continue;
                }
                Set<SelectionKey> selected = selector.selectedKeys();
                printSelectedKeys(selected);
                Iterator<SelectionKey> it = selected.iterator();
                while (it.hasNext()) {
                    dispatch(it.next());
                }
                selected.clear();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void dispatch(SelectionKey key) {
        Runnable r = (Runnable) key.attachment();
        if (r != null) {
            r.run();
        }
    }

    private void printSelectedKeys(Set<SelectionKey> selected) {
    }

    private class Acceptor implements Runnable {

        Selector[] selectors = new Selector[2];

        int next = 0;

        public Acceptor() throws IOException {
            Executor executor = Executors.newFixedThreadPool(selectors.length);
            for (int i = 0; i < selectors.length; i++) {
                selectors[i] = Selector.open();
                Reactor subReactor = new Reactor(selectors[i]);
                executor.execute(subReactor);
            }
        }

        @Override
        public void run() {
            try {
                SocketChannel c = serverSocket.accept();
                System.out.printf("  server established connection: %d
", c.socket().getPort());
                if (c != null) {
                    new Handler(selectors[next], c);
                }
                if (++next == selectors.length) {
                    next = 0;
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

我选择让 Acceptor 创建 subReactor(s)，并通过 Round Robin 的方式将连接平均分配给 subReactors。 每个 subReactor 都运行在一个独立线程中。

代码24行选择了带超时的 select 方法，否则无论我怎样调用 wakeup，程序总是有概率卡死，原因不明。