zk系列-1.客户端数据请求和接受

GetData

// 同步调用
    public byte[] getData(final String path, Watcher watcher, Stat stat) throws KeeperException, InterruptedException {
     
        final String clientPath = path;
        PathUtils.validatePath(clientPath);

        // the watch contains the un-chroot path
        WatchRegistration wcb = null;
        if (watcher != null) {
     
            wcb = new DataWatchRegistration(watcher, clientPath);
        }

        final String serverPath = prependChroot(clientPath);

        RequestHeader h = new RequestHeader();
        h.setType(ZooDefs.OpCode.getData);
        GetDataRequest request = new GetDataRequest();
        request.setPath(serverPath);
        request.setWatch(watcher != null);
        GetDataResponse response = new GetDataResponse();
        ReplyHeader r = cnxn.submitRequest(h, request, response, wcb);
        if (r.getErr() != 0) {
     
            throw KeeperException.create(KeeperException.Code.get(r.getErr()), clientPath);
        }
        if (stat != null) {
     
            DataTree.copyStat(response.getStat(), stat);
        }
        return response.getData();
    }
    // DataCallback 异步调用
    public void getData(final String path, Watcher watcher, DataCallback cb, Object ctx) {
     
        //.....省略部分代码,和上面同步调用相同
        cnxn.queuePacket(h, new ReplyHeader(), request, response, cb, clientPath, serverPath, ctx, wcb);
    }
    
// 实现 Record,通过jute实现序列化和反序列化
public class RequestHeader implements Record {
     
    private int xid;
    private int type;
}
	
public class GetDataRequest implements Record {
     
    private String path;
    private boolean watch;
}

public class GetDataResponse implements Record {
     
    private byte[] data;
    private Stat stat;
    public void serialize(OutputArchive a_, String tag) throws IOException {
     
        a_.startRecord(this, tag);
        a_.writeBuffer(this.data, "data");
        a_.writeRecord(this.stat, "stat");
        a_.endRecord(this, tag);
    }

    public void deserialize(InputArchive a_, String tag) throws IOException {
     
        a_.startRecord(tag);
        this.data = a_.readBuffer("data");
        this.stat = new Stat();
        a_.readRecord(this.stat, "stat");
        a_.endRecord(tag);
    }
}

GetData封装了请求和返回的基本数据结构。cnxn.queuePacket(h, new ReplyHeader(), request, response, cb, clientPath, serverPath, ctx, wcb);是核心方法,具体类见ClientCnxncnxn是在ZooKeeper对象创建时完成初始化。

    public ZooKeeper( String connectString, int sessionTimeout,Watcher watcher,boolean canBeReadOnly,
            HostProvider aHostProvider, ZKClientConfig clientConfig) throws IOException {
     

        if (clientConfig == null) {
     
            clientConfig = new ZKClientConfig();
        }
        this.clientConfig = clientConfig;
        // 默认的watch管理器
        watchManager = defaultWatchManager();
        watchManager.defaultWatcher = watcher;
        ConnectStringParser connectStringParser = new ConnectStringParser(connectString);
        hostProvider = aHostProvider;

        cnxn = createConnection(connectStringParser.getChrootPath(), hostProvider, sessionTimeout, this,
                watchManager, getClientCnxnSocket(), canBeReadOnly);
        cnxn.start();
    }

先分析同步调用

    public ReplyHeader submitRequest(RequestHeader h, Record request, Record response, WatchRegistration watchRegistration,
            WatchDeregistration watchDeregistration) throws InterruptedException {
     
        ReplyHeader r = new ReplyHeader();
        ClientCnxn.Packet packet = queuePacket(h, r, request, response, null, null, null, null, watchRegistration, watchDeregistration);
        // 把request,response封装成Packet后,放入到outgoingQueue中,不断循环阻塞等待结果返回,异步的getData packect入队列后就立刻返回了。
        synchronized (packet) {
     
        	// 设置超时时间则等待超时,否则一直等待。超时时间通过 zookeeper.request.timeout 指定
            if (requestTimeout > 0) {
     
                // Wait for request completion with timeout
                waitForPacketFinish(r, packet);
            } else {
     
                // Wait for request completion infinitely
                while (!packet.finished) {
     
                    packet.wait();
                }
            }
        }
        if (r.getErr() == Code.REQUESTTIMEOUT.intValue()) {
     
            sendThread.cleanAndNotifyState();
        }
        return r;
    }
	public class ReplyHeader implements Record {
     
 	   private int xid;
   	   private long zxid;
       private int err;
 	}
    private void waitForPacketFinish(ReplyHeader r, Packet packet) throws InterruptedException {
     
        long waitStartTime = Time.currentElapsedTime();
        while (!packet.finished) {
     
            // wait等待被唤醒,或者超时
            packet.wait(requestTimeout);
            // 超时被唤醒,置为超时
            if (!packet.finished && ((Time.currentElapsedTime() - waitStartTime) >= requestTimeout)) {
     
                LOG.error("Timeout error occurred for the packet '{}'.", packet);
                r.setErr(Code.REQUESTTIMEOUT.intValue());
                break;
            }
        }
    }
public ClientCnxn.Packet queuePacket(RequestHeader h, ReplyHeader r, Record request, Record response, AsyncCallback cb,
        String clientPath, String serverPath, Object ctx, WatchRegistration watchRegistration, WatchDeregistration watchDeregistration) {
     
    ClientCnxn.Packet packet = null;

    // Note that we do not generate the Xid for the packet yet. It is
    // generated later at send-time, by an implementation of ClientCnxnSocket::doIO(),
    // where the packet is actually sent.
    packet = new ClientCnxn.Packet(h, r, request, response, watchRegistration);
    packet.cb = cb;
    packet.ctx = ctx;
    packet.clientPath = clientPath;
    packet.serverPath = serverPath;
    packet.watchDeregistration = watchDeregistration;
    // The synchronized block here is for two purpose:
    // 1. synchronize with the final cleanup() in SendThread.run() to avoid race
    // 2. synchronized against each packet. So if a closeSession packet is added,
    // later packet will be notified.
    synchronized (state) {
     
        if (!state.isAlive() || closing) {
     
            conLossPacket(packet);
        } else {
     
            // If the client is asking to close the session then
            // mark as closing
            if (h.getType() == OpCode.closeSession) {
     
                closing = true;
            }
            // private final LinkedBlockingDeque outgoingQueue = new LinkedBlockingDeque(); 
            outgoingQueue.add(packet);
        }
    }
    sendThread.getClientCnxnSocket().packetAdded();
    return packet;
}

重要的几个对象

    /**
     * These are the packets that have been sent and are waiting for a response.
     */
    private final Queue<Packet> pendingQueue = new ArrayDeque<>();

    /**
     * These are the packets that need to be sent.
     */
    private final LinkedBlockingDeque<Packet> outgoingQueue = new LinkedBlockingDeque<Packet>();
    
    // 不断发送消息,重连
    final SendThread sendThread;

    final EventThread eventThread;

@Override
public void run() {
     
    clientCnxnSocket.introduce(this, sessionId, outgoingQueue);
    clientCnxnSocket.updateNow();
    clientCnxnSocket.updateLastSendAndHeard();
    int to;
    long lastPingRwServer = Time.currentElapsedTime();
    final int MAX_SEND_PING_INTERVAL = 10000; //10 seconds
    InetSocketAddress serverAddress = null;
    while (state.isAlive()) {
     
        try {
     
        	// 如果断开,重连
            if (!clientCnxnSocket.isConnected()) {
     
                // don't re-establish connection if we are closing
                if (closing) {
     
                    break;
                }
                if (rwServerAddress != null) {
     
                    serverAddress = rwServerAddress;
                    rwServerAddress = null;
                } else {
     
                    serverAddress = hostProvider.next(1000);
                }
                startConnect(serverAddress);
                clientCnxnSocket.updateLastSendAndHeard();
            }

            if (state.isConnected()) {
     
                // determine whether we need to send an AuthFailed event.
                if (zooKeeperSaslClient != null) {
     
                   。。。。。
                }
                to = readTimeout - clientCnxnSocket.getIdleRecv();
            } else {
     
                to = connectTimeout - clientCnxnSocket.getIdleRecv();
            }

            if (to <= 0) {
     
                String warnInfo = String.format(
                    "Client session timed out, have not heard from server in %dms for session id 0x%s",
                    clientCnxnSocket.getIdleRecv(),
                    Long.toHexString(sessionId));
                LOG.warn(warnInfo);
                throw new SessionTimeoutException(warnInfo);
            }
            if (state.isConnected()) {
     
                //1000(1 second) is to prevent race condition missing to send the second ping
                //also make sure not to send too many pings when readTimeout is small
                int timeToNextPing = readTimeout / 2
                                     - clientCnxnSocket.getIdleSend()
                                     - ((clientCnxnSocket.getIdleSend() > 1000) ? 1000 : 0);
                //send a ping request either time is due or no packet sent out within MAX_SEND_PING_INTERVAL 10s。
                if (timeToNextPing <= 0 || clientCnxnSocket.getIdleSend() > MAX_SEND_PING_INTERVAL) {
     
                    sendPing();
                    clientCnxnSocket.updateLastSend();
                } else {
     
                    if (timeToNextPing < to) {
     
                        to = timeToNextPing;
                    }
                }
            }

            // If we are in read-only mode, seek for read/write server
            if (state == States.CONNECTEDREADONLY) {
     
                long now = Time.currentElapsedTime();
                int idlePingRwServer = (int) (now - lastPingRwServer);
                if (idlePingRwServer >= pingRwTimeout) {
     
                    lastPingRwServer = now;
                    idlePingRwServer = 0;
                    pingRwTimeout = Math.min(2 * pingRwTimeout, maxPingRwTimeout);
                    pingRwServer();
                }
                to = Math.min(to, pingRwTimeout - idlePingRwServer);
            }

            clientCnxnSocket.doTransport(to, pendingQueue, ClientCnxn.this);
        } catch (Throwable e) {
     
            if (closing) {
     
                // closing so this is expected
                LOG.warn(
                    "An exception was thrown while closing send thread for session 0x{}.",
                    Long.toHexString(getSessionId()),
                    e);
                break;
            } else {
     
                LOG.warn(
                    "Session 0x{} for sever {}, Closing socket connection. "
                        + "Attempting reconnect except it is a SessionExpiredException.",
                    Long.toHexString(getSessionId()),
                    serverAddress,
                    e);

                // At this point, there might still be new packets appended to outgoingQueue.
                // they will be handled in next connection or cleared up if closed.
                cleanAndNotifyState();
            }
        }
    }

    synchronized (state) {
     
        // When it comes to this point, it guarantees that later queued
        // packet to outgoingQueue will be notified of death.
        cleanup();
    }
    clientCnxnSocket.close();
    if (state.isAlive()) {
     
        eventThread.queueEvent(new WatchedEvent(Event.EventType.None, Event.KeeperState.Disconnected, null));
    }
    eventThread.queueEvent(new WatchedEvent(Event.EventType.None, Event.KeeperState.Closed, null));
    ZooTrace.logTraceMessage(
        LOG,
        ZooTrace.getTextTraceLevel(),
        "SendThread exited loop for session: 0x" + Long.toHexString(getSessionId()));
}

基于Netty的Socket连接,ClientCnxnSocketNetty

    @Override
    void doTransport(
        int waitTimeOut,
        Queue<Packet> pendingQueue,
        ClientCnxn cnxn) throws IOException, InterruptedException {
     
        try {
     
            if (!firstConnect.await(waitTimeOut, TimeUnit.MILLISECONDS)) {
     
                return;
            }
            Packet head = null;
            if (needSasl.get()) {
     
                if (!waitSasl.tryAcquire(waitTimeOut, TimeUnit.MILLISECONDS)) {
     
                    return;
                }
            } else {
     
            // 在outgoingQueue中poll出一个待发送的Packet。
                head = outgoingQueue.poll(waitTimeOut, TimeUnit.MILLISECONDS);
            }
            // check if being waken up on closing.
            if (!sendThread.getZkState().isAlive()) {
     
                // adding back the packet to notify of failure in conLossPacket().
                addBack(head);
                return;
            }
            // channel disconnection happened
            if (disconnected.get()) {
     
                addBack(head);
                throw new EndOfStreamException("channel for sessionid 0x" + Long.toHexString(sessionId) + " is lost");
            }
            if (head != null) {
     
                doWrite(pendingQueue, head, cnxn);
            }
        } finally {
     
            updateNow();
        }
    }

    private void doWrite(Queue<Packet> pendingQueue, Packet p, ClientCnxn cnxn) {
     
        updateNow();
        boolean anyPacketsSent = false;
        // 不断的在outgoingQueue 拿出Packet,然后发送,放到pendingQueue中
        while (true) {
     
            if (p != WakeupPacket.getInstance()) {
     
                if ((p.requestHeader != null)
                    && (p.requestHeader.getType() != ZooDefs.OpCode.ping)
                    && (p.requestHeader.getType() != ZooDefs.OpCode.auth)) {
     
                    p.requestHeader.setXid(cnxn.getXid());
                    synchronized (pendingQueue) {
     
                        pendingQueue.add(p);
                    }
                }
                sendPktOnly(p);
                anyPacketsSent = true;
            }
            if (outgoingQueue.isEmpty()) {
     
                break;
            }
            p = outgoingQueue.remove();
        }
        // TODO: maybe we should flush in the loop above every N packets/bytes?
        // But, how do we determine the right value for N ...
        if (anyPacketsSent) {
     
            channel.flush();
        }
    }

    /**
     * Sends a packet to the remote peer but does not flush() the channel.
     * @param p packet to send.
     * @return a ChannelFuture that will complete when the write operation
     *         succeeds or fails.
     */
    private ChannelFuture sendPktOnly(Packet p) {
     
        return sendPkt(p, false);
    }
    private ChannelFuture sendPkt(Packet p, boolean doFlush) {
     
        // Assuming the packet will be sent out successfully. Because if it fails,
        // the channel will close and clean up queues.
        p.createBB();
        updateLastSend();
        final ByteBuf writeBuffer = Unpooled.wrappedBuffer(p.bb);
        // 发送给服务器
        final ChannelFuture result = doFlush ? channel.writeAndFlush(writeBuffer) : channel.write(writeBuffer);
        result.addListener(onSendPktDoneListener);
        return result;
    }

        public void createBB() {
     
            try {
     
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
                BinaryOutputArchive boa = BinaryOutputArchive.getArchive(baos);
                boa.writeInt(-1, "len"); // We'll fill this in later
                if (requestHeader != null) {
     
                // 基于jute的序列化
                    requestHeader.serialize(boa, "header");
                }
                if (request instanceof ConnectRequest) {
     
                    request.serialize(boa, "connect");
                    // append "am-I-allowed-to-be-readonly" flag
                    boa.writeBool(readOnly, "readOnly");
                } else if (request != null) {
     
                    request.serialize(boa, "request");
                }
                baos.close();
                // ByteBuffer bb;
                this.bb = ByteBuffer.wrap(baos.toByteArray());
                this.bb.putInt(this.bb.capacity() - 4);
                this.bb.rewind();
            } catch (IOException e) {
     
                LOG.warn("Unexpected exception", e);
            }
        }

截止到目前,数据已经成功的发送给服务器。

下面看一下数据的接受。重点类ClientCnxnSocketNetty。基于socket的通信是双向的。当服务端发送消息到客户端时会唤起channelRead0

        @Override
        protected void channelRead0(ChannelHandlerContext ctx, ByteBuf buf) throws Exception {
     
            updateNow();
            while (buf.isReadable()) {
     
                if (incomingBuffer.remaining() > buf.readableBytes()) {
     
                    int newLimit = incomingBuffer.position() + buf.readableBytes();
                    incomingBuffer.limit(newLimit);
                }
                buf.readBytes(incomingBuffer);
                incomingBuffer.limit(incomingBuffer.capacity());

                if (!incomingBuffer.hasRemaining()) {
     
                    incomingBuffer.flip();
                    if (incomingBuffer == lenBuffer) {
     
                        recvCount.getAndIncrement();
                        readLength();
                    } else if (!initialized) {
     
                        readConnectResult();
                        lenBuffer.clear();
                        incomingBuffer = lenBuffer;
                        initialized = true;
                        updateLastHeard();
                    } else {
     
                    // 调用了SendThread的readResponse
                        sendThread.readResponse(incomingBuffer);
                        lenBuffer.clear();
                        incomingBuffer = lenBuffer;
                        updateLastHeard();
                    }
                }
            }
            wakeupCnxn();
            // Note: SimpleChannelInboundHandler releases the ByteBuf for us
            // so we don't need to do it.
        }
void readResponse(ByteBuffer incomingBuffer) throws IOException {
     
            ByteBufferInputStream bbis = new ByteBufferInputStream(incomingBuffer);
            BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis);
            ReplyHeader replyHdr = new ReplyHeader();
            // 反序列化 header
            replyHdr.deserialize(bbia, "header");
            
            switch (replyHdr.getXid()) {
     
            case PING_XID:
                LOG.debug("Got ping response for session id: 0x{} after {}ms.",
                    Long.toHexString(sessionId),
                    ((System.nanoTime() - lastPingSentNs) / 1000000));
                return;
            case AUTHPACKET_XID:
                LOG.debug("Got auth session id: 0x{}", Long.toHexString(sessionId));
                if (replyHdr.getErr() == KeeperException.Code.AUTHFAILED.intValue()) {
     
                    state = States.AUTH_FAILED;
                    eventThread.queueEvent(new WatchedEvent(Watcher.Event.EventType.None,
                        Watcher.Event.KeeperState.AuthFailed, null));
                    eventThread.queueEventOfDeath();
                }
              return;
            // 事件触发响应
            case NOTIFICATION_XID:
                LOG.debug("Got notification session id: 0x{}",
                    Long.toHexString(sessionId));
                // 反序列化事件对象。
                WatcherEvent event = new WatcherEvent();
                event.deserialize(bbia, "response");

                // convert from a server path to a client path
                if (chrootPath != null) {
     
                    String serverPath = event.getPath();
                    if (serverPath.compareTo(chrootPath) == 0) {
     
                        event.setPath("/");
                    } else if (serverPath.length() > chrootPath.length()) {
     
                        event.setPath(serverPath.substring(chrootPath.length()));
                     } else {
     
                         LOG.warn("Got server path {} which is too short for chroot path {}.",
                             event.getPath(), chrootPath);
                     }
                }
                // 包装事件对象,放入eventThread。
                // private final LinkedBlockingQueue waitingEvents = new LinkedBlockingQueue();
                WatchedEvent we = new WatchedEvent(event);
                LOG.debug("Got {} for session id 0x{}", we, Long.toHexString(sessionId));
                eventThread.queueEvent(we);
                return;
            default:
                break;
            }

            // If SASL authentication is currently in progress, construct and
            // send a response packet immediately, rather than queuing a
            // response as with other packets.
            if (tunnelAuthInProgress()) {
     
                GetSASLRequest request = new GetSASLRequest();
                request.deserialize(bbia, "token");
                zooKeeperSaslClient.respondToServer(request.getToken(), ClientCnxn.this);
                return;
            }

			// 下面处理正常的请求,getData等,客户端主动触发的请求。
            Packet packet;
            synchronized (pendingQueue) {
     
                if (pendingQueue.size() == 0) {
     
                    throw new IOException("Nothing in the queue, but got " + replyHdr.getXid());
                }
                packet = pendingQueue.remove();
            }
            /*
             * Since requests are processed in order, we better get a response
             * to the first request!
             */
            try {
     
            // 保证请求是按照顺序处理的
                if (packet.requestHeader.getXid() != replyHdr.getXid()) {
     
                    packet.replyHeader.setErr(KeeperException.Code.CONNECTIONLOSS.intValue());
                    throw new IOException("Xid out of order. Got Xid " + replyHdr.getXid()
                                          + " with err " + replyHdr.getErr()
                                          + " expected Xid " + packet.requestHeader.getXid()
                                          + " for a packet with details: " + packet);
                }

                packet.replyHeader.setXid(replyHdr.getXid());
                packet.replyHeader.setErr(replyHdr.getErr());
                packet.replyHeader.setZxid(replyHdr.getZxid());
                if (replyHdr.getZxid() > 0) {
     
                    lastZxid = replyHdr.getZxid();
                }
                if (packet.response != null && replyHdr.getErr() == 0) {
     
                    packet.response.deserialize(bbia, "response");
                }

                LOG.debug("Reading reply session id: 0x{}, packet:: {}", Long.toHexString(sessionId), packet);
            } finally {
     
                finishPacket(packet);
            }
        }
    // @VisibleForTesting
    protected void finishPacket(Packet p) {
     
        int err = p.replyHeader.getErr();
        if (p.watchRegistration != null) {
     
            p.watchRegistration.register(err);
        }
        // Add all the removed watch events to the event queue, so that the
        // clients will be notified with 'Data/Child WatchRemoved' event type.
        if (p.watchDeregistration != null) {
     
            Map<EventType, Set<Watcher>> materializedWatchers = null;
            try {
     
                materializedWatchers = p.watchDeregistration.unregister(err);
                for (Entry<EventType, Set<Watcher>> entry : materializedWatchers.entrySet()) {
     
                    Set<Watcher> watchers = entry.getValue();
                    if (watchers.size() > 0) {
     
                        queueEvent(p.watchDeregistration.getClientPath(), err, watchers, entry.getKey());
                        // ignore connectionloss when removing from local
                        // session
                        p.replyHeader.setErr(Code.OK.intValue());
                    }
                }
            } catch (KeeperException.NoWatcherException nwe) {
     
                p.replyHeader.setErr(nwe.code().intValue());
            } catch (KeeperException ke) {
     
                p.replyHeader.setErr(ke.code().intValue());
            }
        }

		// AsyncCallback cb; 非异步直接notify,异步放入事件处理线程中。
        if (p.cb == null) {
     
            synchronized (p) {
     
                p.finished = true;
                // 通知等待线程,唤醒submitRequest的packet.wait();
                p.notifyAll();
            }
        } else {
     
            p.finished = true;
            eventThread.queuePacket(p);
        }
    
 
  

截止到目前,数据已经被接收到了。目前还有两个问题:
1.eventThread怎么处理事件的。
2.eventThread怎么实现异步回调。
见 zk系列-2.watch机制及异步回调的原理

你可能感兴趣的:(开源框架,zookeeper,zk,java,websocket)