ClientCnxn是客户端的类：该类管理zk客户端的socket io，维持一个可用服务器的列表。
//ClientCnxn类 private final LinkedList<Packet> outgoingQueue = new LinkedList<Packet>(); //待发送 private final LinkedList<Packet> pendingQueue = new LinkedList<Packet>(); //发送后等待响应 final SendThread sendThread; final EventThread eventThread;
Packet封装了请求、响应以及回调等。
static class Packet { //省略其他代码 RequestHeader requestHeader; ReplyHeader replyHeader; Record request; Record response; ByteBuffer bb; String clientPath; String serverPath; boolean finished; AsyncCallback cb; Object ctx; WatchRegistration watchRegistration; public boolean readOnly; Packet(RequestHeader requestHeader, ReplyHeader replyHeader, Record request, Record response, WatchRegistration watchRegistration) { this(requestHeader, replyHeader, request, response, watchRegistration, false); } Packet(RequestHeader requestHeader, ReplyHeader replyHeader, Record request, Record response, WatchRegistration watchRegistration, boolean readOnly) { this.requestHeader = requestHeader; this.replyHeader = replyHeader; this.request = request; this.response = response; this.readOnly = readOnly; this.watchRegistration = watchRegistration; } }
ClientCnxn类中有SendThread和EventThread两个线程，SendThread负责io（发送和接收），EventThread负责事件处理。
Packet在outgoingQueue和pendingQueue之间流转：
首先，调用线程把Packet放在outgoingQueue中，SendThread从outgoingQueue中取出Packet并发送，然后把该Packet放入pendingQueue中，等待响应，当响应来到时，解析响应，并从pendingQueue中删除Packet。
示例代码：
public class MyZK { public static void main(String[] args) throws IOException, KeeperException, InterruptedException { ZooKeeper zk = new ZooKeeper("127.0.0.1:2181", 60000000, new MyWatcher()); List<ACL> list = new ArrayList<ACL>(1); list.add(new ACL(Perms.ALL, Ids.ANYONE_ID_UNSAFE)); //CreateResponse String create = zk.create("/zhang/hello", "hello".getBytes(), list, CreateMode.PERSISTENT); System.out.println(create); //GetDataResponse
// 添加了watch。删除该节点的时候，服务端会发通知
byte[] data = zk.getData("/zhang/hello", true, null); System.out.println(new String(data)); Stat stat = zk.exists("/zhang/hello", false); System.out.println(stat); zk.delete("/zhang/hello", -1); } static class MyWatcher implements Watcher { public void process(WatchedEvent event) { System.out.println(event); } } }
getData 调用栈：
// ZooKeeper.getData public byte[] getData(final String path, Watcher watcher, Stat stat) throws KeeperException, InterruptedException { final String clientPath = path; PathUtils.validatePath(clientPath); // the watch contains the un-chroot path WatchRegistration wcb = null; if (watcher != null) { wcb = new DataWatchRegistration(watcher, clientPath); } final String serverPath = prependChroot(clientPath); RequestHeader h = new RequestHeader(); //请求头 h.setType(ZooDefs.OpCode.getData); GetDataRequest request = new GetDataRequest(); //请求 request.setPath(serverPath); request.setWatch(watcher != null); GetDataResponse response = new GetDataResponse(); //响应 ReplyHeader r = cnxn.submitRequest(h, request, response, wcb); //发送请求，阻塞，等待响应 if (r.getErr() != 0) { throw KeeperException.create(KeeperException.Code.get(r.getErr()), clientPath); } if (stat != null) { DataTree.copyStat(response.getStat(), stat); } return response.getData(); }
调用线程把请求放入outgoingQueue队列中，然后阻塞：
public ReplyHeader submitRequest(RequestHeader h, Record request, Record response, WatchRegistration watchRegistration) throws InterruptedException { ReplyHeader r = new ReplyHeader(); //创建Packet，并放入outgoingQueue中 Packet packet = queuePacket(h, r, request, response, null, null, null, null, watchRegistration); //先获取锁 synchronized (packet) { while (!packet.finished) { //等待，并释放锁 packet.wait(); } } return r; }
SendThread负责io操作，发送请求，接收响应。
发送：把outgoingQueue中的packet发送出去，然后把packet放到pendingQueue中，等待响应。
接收：解析响应，从pendingQueue中删除packet。
// ClientCnxnSocketNIO.doIO void doIO(List<Packet> pendingQueue, LinkedList<Packet> outgoingQueue, ClientCnxn cnxn) throws InterruptedException, IOException { SocketChannel sock = (SocketChannel) sockKey.channel(); if (sock == null) { throw new IOException("Socket is null!"); } if (sockKey.isReadable()) { //读取响应 int rc = sock.read(incomingBuffer); if (rc < 0) { throw new EndOfStreamException( "Unable to read additional data from server sessionid 0x" + Long.toHexString(sessionId) + ", likely server has closed socket"); } if (!incomingBuffer.hasRemaining()) { incomingBuffer.flip(); if (incomingBuffer == lenBuffer) { recvCount++; readLength(); } else if (!initialized) { readConnectResult(); enableRead(); if (findSendablePacket(outgoingQueue, cnxn.sendThread.clientTunneledAuthenticationInProgress()) != null) { enableWrite(); } lenBuffer.clear(); incomingBuffer = lenBuffer; updateLastHeard(); initialized = true; } else { //解析响应 sendThread.readResponse(incomingBuffer); lenBuffer.clear(); incomingBuffer = lenBuffer; updateLastHeard(); } } } if (sockKey.isWritable()) { //发送请求 synchronized(outgoingQueue) { Packet p = findSendablePacket(outgoingQueue, cnxn.sendThread.clientTunneledAuthenticationInProgress()); if (p != null) { updateLastSend(); // If we already started writing p, p.bb will already exist if (p.bb == null) { if ((p.requestHeader != null) && (p.requestHeader.getType() != OpCode.ping) && (p.requestHeader.getType() != OpCode.auth)) { p.requestHeader.setXid(cnxn.getXid()); } p.createBB(); } sock.write(p.bb); if (!p.bb.hasRemaining()) { sentCount++; //从outgoingQueue中删除packet outgoingQueue.removeFirstOccurrence(p); if (p.requestHeader != null && p.requestHeader.getType() != OpCode.ping && p.requestHeader.getType() != OpCode.auth) { synchronized (pendingQueue) { //把packet加入到pendingQueue中 pendingQueue.add(p); } } } } if (outgoingQueue.isEmpty()) { disableWrite(); } else if (!initialized && p != null && !p.bb.hasRemaining()) { disableWrite(); } else { // Just in case enableWrite(); } } } }
解析响应：
//SendThread类 void readResponse(ByteBuffer incomingBuffer) throws IOException { //省略其他代码 ByteBufferInputStream bbis = new ByteBufferInputStream(incomingBuffer); BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis); ReplyHeader replyHdr = new ReplyHeader(); replyHdr.deserialize(bbia, "header"); // -1表示notification if (replyHdr.getXid() == -1) { //反序列化，设置WatcherEvent对象 WatcherEvent event = new WatcherEvent(); event.deserialize(bbia, "response"); // convert from a server path to a client path if (chrootPath != null) { String serverPath = event.getPath(); if(serverPath.compareTo(chrootPath)==0) event.setPath("/"); else if (serverPath.length() > chrootPath.length()) event.setPath(serverPath.substring(chrootPath.length())); else { LOG.warn("Got server path " + event.getPath() + " which is too short for chroot path " + chrootPath); } } WatchedEvent we = new WatchedEvent(event); //放入eventThread的队列 eventThread.queueEvent(we); return; } //正常的响应数据 Packet packet; synchronized (pendingQueue) { if (pendingQueue.size() == 0) { throw new IOException("Nothing in the queue, but got " + replyHdr.getXid()); } packet = pendingQueue.remove(); } /* * Since requests are processed in order, we better get a response * to the first request! */ try { if (packet.requestHeader.getXid() != replyHdr.getXid()) { packet.replyHeader.setErr(KeeperException.Code.CONNECTIONLOSS.intValue()); throw new IOException("Xid out of order."); } packet.replyHeader.setXid(replyHdr.getXid()); packet.replyHeader.setErr(replyHdr.getErr()); packet.replyHeader.setZxid(replyHdr.getZxid()); if (replyHdr.getZxid() > 0) { lastZxid = replyHdr.getZxid(); } if (packet.response != null && replyHdr.getErr() == 0) { //反序列化设置response内容 packet.response.deserialize(bbia, "response"); } } finally { finishPacket(packet); } }
调用线程现在还阻塞着呢，需要有人唤醒：
private void finishPacket(Packet p) { if (p.watchRegistration != null) { p.watchRegistration.register(p.replyHeader.getErr()); } if (p.cb == null) { synchronized (p) { p.finished = true; //唤醒调用线程 p.notifyAll(); } } else { p.finished = true; eventThread.queuePacket(p); } }
问题：
SendThread是按顺序发送outgoingQueue中的Packet，然后放入pendingQueue中，但是接收到的响应未必是按顺序的（网络环境中先发的数据包未必先到达吧），
zk就这么自信收到的响应和pendingQueue中的顺序一致？