服务端启动时，就启动线程通过NIO监听网络端口。每个连接都会有一个上下文环境对象，当接收到请求后，会在上下文环境对象中进行处理。
服务端启动线程，监听网络端口，（NIOServerCnxn.Factory）：
1 static public class Factory extends Thread { 2 static { 3 //设置全局的异常处理 4 Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { 5 public void uncaughtException(Thread t, Throwable e) { 6 LOG.error("Thread " + t + " died", e); 7 } 8 }); 9 /** 10 * jvm早期的nullpoint bug 11 * http://bugs.sun.com/view_bug.do?bug_id=6427854 12 */ 13 try { 14 Selector.open().close(); 15 } catch(IOException ie) { 16 LOG.error("Selector failed to open", ie); 17 } 18 } 19 //服务器通道 20 final ServerSocketChannel ss; 21 //选择器 22 final Selector selector = Selector.open(); 23 ZooKeeperServer zks; 24 25 final ByteBuffer directBuffer = ByteBuffer.allocateDirect(64 * 1024); 26 //所有的上下文环境 27 final HashSet<NIOServerCnxn> cnxns = new HashSet<NIOServerCnxn>(); 28 //ip对应的上下文环境 29 final HashMap<InetAddress, Set<NIOServerCnxn>> ipMap = 30 new HashMap<InetAddress, Set<NIOServerCnxn>>( ); 31 int maxClientCnxns = 10; 32 33 public Factory(InetSocketAddress addr, int maxcc) throws IOException { 34 setDaemon(true); 35 //单个client链接最大数 36 maxClientCnxns = maxcc; 37 //创建服务器通道 38 this.ss = ServerSocketChannel.open(); 39 ss.socket().setReuseAddress(true); 40 //绑定端口 41 ss.socket().bind(addr); 42 //设置通道为非阻塞通道 43 ss.configureBlocking(false); 44 //把通道注册到选择器中 45 ss.register(selector, SelectionKey.OP_ACCEPT); 46 } 47 public void run() { 48 while (!ss.socket().isClosed()) { 49 try { 50 //选择一组键 51 selector.select(1000); 52 Set<SelectionKey> selected; 53 synchronized (this) { 54 selected = selector.selectedKeys(); 55 } 56 ArrayList<SelectionKey> selectedList = new ArrayList<SelectionKey>( 57 selected); 58 Collections.shuffle(selectedList); 59 for (SelectionKey k : selectedList) { 60 //如果通道已经准备好接收套接字 61 if ((k.readyOps() & SelectionKey.OP_ACCEPT) != 0) { 62 SocketChannel sc = ((ServerSocketChannel) k 63 .channel()).accept(); 64 InetAddress ia = sc.socket().getInetAddress(); 65 //判断最大连接数 66 int cnxncount = getClientCnxnCount(ia); 67 if (maxClientCnxns > 0 && cnxncount >= maxClientCnxns){ 68 sc.close(); 69 } else { 70 // 配置为非阻塞 71 sc.configureBlocking(false); 72 //把通道注册到选择器中 73 SelectionKey sk = sc.register(selector, 74 SelectionKey.OP_READ); 75 NIOServerCnxn cnxn = createConnection(sc, sk); 76 //给该通道附带一个上下文环境 77 sk.attach(cnxn); 78 addCnxn(cnxn); 79 } 80 } else if ((k.readyOps() & (SelectionKey.OP_READ | SelectionKey.OP_WRITE)) != 0) { 81 //通过上线文件来进行读写 82 NIOServerCnxn c = (NIOServerCnxn) k.attachment(); 83 c.doIO(k); 84 } else { 85 if (LOG.isDebugEnabled()) { 86 LOG.debug("Unexpected ops in select " 87 + k.readyOps()); 88 } 89 } 90 } 91 selected.clear(); 92 catch (Exception e) { 93 LOG.warn("Ignoring exception", e); 94 } 95 } 96 clear(); 97 } 98 //关闭 99 public void shutdown() { 100 try { 101 ss.close(); 102 clear(); 103 this.interrupt(); 104 this.join(); 105 } catch (Exception e) { 106 LOG.warn("Ignoring unexpected exception during shutdown", e); 107 } 108 try { 109 selector.close(); 110 } catch (IOException e) { 111 LOG.warn("Selector closing", e); 112 } 113 if (zks != null) { 114 zks.shutdown(); 115 } 116 } 117 synchronized public void clear() { 118 selector.wakeup(); 119 HashSet<NIOServerCnxn> cnxns; 120 synchronized (this.cnxns) { 121 cnxns = (HashSet<NIOServerCnxn>)this.cnxns.clone(); 122 } 123 // got to clear all the connections that we have in the selector 124 for (NIOServerCnxn cnxn: cnxns) { 125 try { 126 // don't hold this.cnxns lock as deadlock may occur 127 cnxn.close(); 128 } catch (Exception e) { 129 LOG.warn("Ignoring exception closing cnxn sessionid 0x" 130 + Long.toHexString(cnxn.sessionId), e); 131 } 132 } 133 } 134 }
上下文环境主要通过NIO读取请求数据，首先会读取4个字节的请求数据，该数据分为两种情况，一种是真正的数据包长度，一种是命令。如果是真正的数据包长度会按长度读取数据报，进行处理。如果是命令，会根据命令进行处理。
服务端通道上下文，通过NIO进行读写（NIOServerCnxn.doIO）：
1 void doIO(SelectionKey k) throws InterruptedException { 2 try { 3 //如果通道可以读取数据 4 if (k.isReadable()) { 5 //读取数据到缓存中 6 int rc = sock.read(incomingBuffer); 7 //如果读满缓存 8 if (incomingBuffer.remaining() == 0) { 9 boolean isPayload; 10 //如果第一次读取，则先读取长度内容，相应分配缓存；否则读取指定长度的数据内容 11 if (incomingBuffer == lenBuffer) { 12 incomingBuffer.flip(); 13 isPayload = readLength(k); 14 incomingBuffer.clear(); 15 } else { 16 isPayload = true; 17 } 18 if (isPayload) { 19 //读取数据，初始化、读取请求数据封装成packet 20 readPayload(); 21 } 22 } 23 } 24 //如果通道可以读写数据 25 if (k.isWritable()) { 26 //缓存中有数据需要写入 27 if (outgoingBuffers.size() > 0) { 28 //创建bytebuffer 29 ByteBuffer directBuffer = factory.directBuffer; 30 directBuffer.clear(); 31 //从队列中读取数据知道缓存读满 32 for (ByteBuffer b : outgoingBuffers) { 33 if (directBuffer.remaining() < b.remaining()) { 34 b = (ByteBuffer) b.slice().limit( 35 directBuffer.remaining()); 36 } 37 int p = b.position(); 38 directBuffer.put(b); 39 b.position(p); 40 if (directBuffer.remaining() == 0) { 41 break; 42 } 43 } 44 //将数据写入通道 45 directBuffer.flip(); 46 int sent = sock.write(directBuffer); 47 ByteBuffer bb; 48 //从缓存中已经发送的删除数据 49 while (outgoingBuffers.size() > 0) { 50 bb = outgoingBuffers.peek(); 51 int left = bb.remaining() - sent; 52 if (left > 0) { 53 bb.position(bb.position() + sent); 54 break; 55 } 56 sent -= bb.remaining(); 57 outgoingBuffers.remove(); 58 } 59 } 60 61 synchronized(this.factory){ 62 if (outgoingBuffers.size() == 0) { 63 sk.interestOps(sk.interestOps() 64 & (~SelectionKey.OP_WRITE)); 65 } else { 66 sk.interestOps(sk.interestOps() 67 | SelectionKey.OP_WRITE); 68 } 69 } 70 } 71 }catch (IOException e) { 72 close(); 73 } 74 } 75 private void readPayload() throws IOException, InterruptedException { 76 if (incomingBuffer.remaining() != 0) { // have we read length bytes? 77 int rc = sock.read(incomingBuffer); // sock is non-blocking, so ok 78 } 79 if (incomingBuffer.remaining() == 0) { 80 //重置缓存 81 incomingBuffer.flip(); 82 //如果没有进行初始化，首先要初始化；如果已经链接，则读取请求数据，封装成packet 83 if (!initialized) { 84 readConnectRequest(); 85 } else { 86 readRequest(); 87 } 88 //重置 89 lenBuffer.clear(); 90 incomingBuffer = lenBuffer; 91 } 92 }
服务端通道上下文，读取长度，如果是命令，另起线程处理命令操作（NIOServerCnxn.readLength）
1 private boolean readLength(SelectionKey k) throws IOException { 2 //如果是请求数据，根据长度分配缓存；如果是命令，执行相应命令。 3 int len = lenBuffer.getInt(); 4 if (!initialized && checkFourLetterWord(k, len)) { 5 return false; 6 } 7 if (len < 0 || len > BinaryInputArchive.maxBuffer) { 8 throw new IOException("Len error " + len); 9 } 10 if (zk == null) { 11 throw new IOException("ZooKeeperServer not running"); 12 } 13 incomingBuffer = ByteBuffer.allocate(len); 14 return true; 15 } 16 private boolean checkFourLetterWord(final SelectionKey k, final int len) 17 throws IOException 18 { 19 //获取命令 20 String cmd = cmd2String.get(len); 21 /** cancel the selection key to remove the socket handling 22 * from selector. This is to prevent netcat problem wherein 23 * netcat immediately closes the sending side after sending the 24 * commands and still keeps the receiving channel open. 25 * The idea is to remove the selectionkey from the selector 26 * so that the selector does not notice the closed read on the 27 * socket channel and keep the socket alive to write the data to 28 * and makes sure to close the socket after its done writing the data 29 */ 30 if (k != null) { 31 try { 32 k.cancel(); 33 } catch(Exception e) { 34 LOG.error("Error cancelling command selection key ", e); 35 } 36 } 37 //根据命令类型，执行相应内容 38 final PrintWriter pwriter = new PrintWriter( 39 new BufferedWriter(new SendBufferWriter())); 40 if (len == ruokCmd) { 41 RuokCommand ruok = new RuokCommand(pwriter); 42 ruok.start(); 43 return true; 44 } else if (len == getTraceMaskCmd) { 45 TraceMaskCommand tmask = new TraceMaskCommand(pwriter); 46 tmask.start(); 47 return true; 48 } else if (len == setTraceMaskCmd) { 49 int rc = sock.read(incomingBuffer); 50 if (rc < 0) { 51 throw new IOException("Read error"); 52 } 53 54 incomingBuffer.flip(); 55 long traceMask = incomingBuffer.getLong(); 56 ZooTrace.setTextTraceLevel(traceMask); 57 SetTraceMaskCommand setMask = new SetTraceMaskCommand(pwriter, traceMask); 58 setMask.start(); 59 return true; 60 } else if (len == enviCmd) { 61 EnvCommand env = new EnvCommand(pwriter); 62 env.start(); 63 return true; 64 } else if (len == confCmd) { 65 ConfCommand ccmd = new ConfCommand(pwriter); 66 ccmd.start(); 67 return true; 68 } else if (len == srstCmd) { 69 StatResetCommand strst = new StatResetCommand(pwriter); 70 strst.start(); 71 return true; 72 } else if (len == crstCmd) { 73 CnxnStatResetCommand crst = new CnxnStatResetCommand(pwriter); 74 crst.start(); 75 return true; 76 } else if (len == dumpCmd) { 77 DumpCommand dump = new DumpCommand(pwriter); 78 dump.start(); 79 return true; 80 } else if (len == statCmd || len == srvrCmd) { 81 StatCommand stat = new StatCommand(pwriter, len); 82 stat.start(); 83 return true; 84 } else if (len == consCmd) { 85 ConsCommand cons = new ConsCommand(pwriter); 86 cons.start(); 87 return true; 88 } else if (len == wchpCmd || len == wchcCmd || len == wchsCmd) { 89 WatchCommand wcmd = new WatchCommand(pwriter, len); 90 wcmd.start(); 91 return true; 92 } 93 return false; 94 }
服务端通道上下文，如果是请求数据，处理请求数据（NIOServerCnxn.readRequestNIOServerCnxn.readConnectRequest）：
1 private void readRequest() throws IOException { 2 //反序列化请求数据 3 InputStream bais = new ByteBufferInputStream(incomingBuffer); 4 BinaryInputArchive bia = BinaryInputArchive.getArchive(bais); 5 RequestHeader h = new RequestHeader(); 6 h.deserialize(bia, "header"); 7 incomingBuffer = incomingBuffer.slice(); 8 if (h.getType() == OpCode.auth) { 9 //如果是认证请求 10 AuthPacket authPacket = new AuthPacket(); 11 ZooKeeperServer.byteBuffer2Record(incomingBuffer, authPacket); 12 String scheme = authPacket.getScheme(); 13 //进行认证 14 AuthenticationProvider ap = ProviderRegistry.getProvider(scheme); 15 if (ap == null 16 || (ap.handleAuthentication(this, authPacket.getAuth()) 17 != KeeperException.Code.OK)) { 18 // 认证失败，返回失败内容 19 ReplyHeader rh = new ReplyHeader(h.getXid(), 0, 20 KeeperException.Code.AUTHFAILED.intValue()); 21 sendResponse(rh, null, null); 22 //关闭链接 23 sendCloseSession(); 24 disableRecv(); 25 } else { 26 //认证成功，返回成功内容 27 ReplyHeader rh = new ReplyHeader(h.getXid(), 0, 28 KeeperException.Code.OK.intValue()); 29 sendResponse(rh, null, null); 30 } 31 return; 32 } else { 33 //如果是请求，提交到zk处理 34 Request si = new Request(this, sessionId, h.getXid(), h.getType(), incomingBuffer, authInfo); 35 si.setOwner(ServerCnxn.me); 36 zk.submitRequest(si); 37 } 38 } 39 40 private void readConnectRequest() throws IOException, InterruptedException { 41 //反序列化链接请求对象 42 BinaryInputArchive bia = BinaryInputArchive 43 .getArchive(new ByteBufferInputStream(incomingBuffer)); 44 ConnectRequest connReq = new ConnectRequest(); 45 connReq.deserialize(bia, "connect"); 46 if (connReq.getLastZxidSeen() > zk.getZKDatabase().getDataTreeLastProcessedZxid()) { 47 throw new CloseRequestException(msg); 48 } 49 sessionTimeout = connReq.getTimeOut(); 50 byte passwd[] = connReq.getPasswd(); 51 //初始化session 52 disableRecv(); 53 if (connReq.getSessionId() != 0) { 54 long clientSessionId = connReq.getSessionId(); 55 factory.closeSessionWithoutWakeup(clientSessionId); 56 setSessionId(clientSessionId); 57 zk.reopenSession(this, sessionId, passwd, sessionTimeout); 58 } else { 59 zk.createSession(this, passwd, sessionTimeout); 60 } 61 initialized = true; 62 }
服务端通道上下文，写返回数据（NIOServerCnxn.sendResponse）
1 synchronized public void sendResponse(ReplyHeader h, Record r, String tag) { 2 try { 3 //序列化返回结果， 4 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 5 BinaryOutputArchive bos = BinaryOutputArchive.getArchive(baos); 6 try { 7 baos.write(fourBytes); 8 bos.writeRecord(h, "header"); 9 if (r != null) { 10 bos.writeRecord(r, tag); 11 } 12 baos.close(); 13 } catch (IOException e) { 14 LOG.error("Error serializing response"); 15 } 16 //写入数据长度 17 byte b[] = baos.toByteArray(); 18 ByteBuffer bb = ByteBuffer.wrap(b); 19 bb.putInt(b.length - 4).rewind(); 20 // 21 sendBuffer(bb); 22 } catch(Exception e) { 23 LOG.warn("Unexpected exception. Destruction averted.", e); 24 } 25 } 26 void sendBuffer(ByteBuffer bb) { 27 try { 28 if (bb != closeConn) { 29 //直接发送数据 30 if ((sk.interestOps() & SelectionKey.OP_WRITE) == 0) { 31 try { 32 sock.write(bb); 33 } catch (IOException e) { 34 // we are just doing best effort right now 35 } 36 } 37 if (bb.remaining() == 0) { 38 packetSent(); 39 return; 40 } 41 } 42 //写入缓存中。 43 synchronized(this.factory){ 44 sk.selector().wakeup(); 45 outgoingBuffers.add(bb); 46 if (sk.isValid()) { 47 sk.interestOps(sk.interestOps() | SelectionKey.OP_WRITE); 48 } 49 } 50 51 } catch(Exception e) { 52 LOG.error("Unexpected Exception: ", e); 53 } 54 }