zookeeper集群模式（十一）zookeeper之Leader与Learner网络交互之LearnerHandler
- LearnerCnxAcceptor
这个是Leader的一个内部类，Leader是通过它来与Learner建立连接的，接下来看看这个类：
class LearnerCnxAcceptor extends ZooKeeperCriticalThread {
private volatile boolean stop = false;
public LearnerCnxAcceptor() {
super("LearnerCnxAcceptor-" + ss.getLocalSocketAddress(), zk
.getZooKeeperServerListener());
}
@Override
public void run() {
try {
while (!stop) {
Socket s = null;
boolean error = false;
try {
s = ss.accept();
// start with the initLimit, once the ack is processed
// in LearnerHandler switch to the syncLimit
s.setSoTimeout(self.tickTime * self.initLimit);
s.setTcpNoDelay(nodelay);
BufferedInputStream is = new BufferedInputStream(
s.getInputStream());
LearnerHandler fh = new LearnerHandler(s, is,
Leader.this);
fh.start();
} catch (SocketException e) {
error = true;
if (stop) {
LOG.info("exception while shutting down acceptor: "
+ e);
// When Leader.shutdown() calls ss.close(),
// the call to accept throws an exception.
// We catch and set stop to true.
stop = true;
} else {
throw e;
}
} catch (SaslException e){
LOG.error("Exception while connecting to quorum learner", e);
error = true;
} catch (Exception e) {
error = true;
throw e;
} finally {
// Don't leak sockets on errors
if (error && s != null && !s.isClosed()) {
try {
s.close();
} catch (IOException e) {
LOG.warn("Error closing socket", e);
}
}
}
}
} catch (Exception e) {
LOG.warn("Exception while accepting follower", e.getMessage());
handleException(this.getName(), e);
}
}
public void halt() {
stop = true;
}
}
Leader(QuorumPeer self,LeaderZooKeeperServer zk) throws IOException {
this.self = self;
this.proposalStats = new BufferStats();
try {
if (self.shouldUsePortUnification() || self.isSslQuorum()) {
boolean allowInsecureConnection = self.shouldUsePortUnification();
if (self.getQuorumListenOnAllIPs()) {
ss = new UnifiedServerSocket(self.getX509Util(), allowInsecureConnection, self.getQuorumAddress().getPort());
} else {
ss = new UnifiedServerSocket(self.getX509Util(), allowInsecureConnection);
}
} else {
if (self.getQuorumListenOnAllIPs()) {
ss = new ServerSocket(self.getQuorumAddress().getPort());
} else {
ss = new ServerSocket();
}
}
ss.setReuseAddress(true);
if (!self.getQuorumListenOnAllIPs()) {
ss.bind(self.getQuorumAddress());
}
} catch (BindException e) {
if (self.getQuorumListenOnAllIPs()) {
LOG.error("Couldn't bind to port " + self.getQuorumAddress().getPort(), e);
} else {
LOG.error("Couldn't bind to " + self.getQuorumAddress(), e);
}
throw e;
}
this.zk = zk;
this.learnerSnapshotThrottler = createLearnerSnapshotThrottler(
maxConcurrentSnapshots, maxConcurrentSnapshotTimeout);
}
分析如下：
1.在创建Leader实例的时候会初始化一个ServerSocket用来跟Follower以及Observer通信，端口是quorumAddress即投票端口
2.等待Learner的连接
3.如果有Learner与Leader建立连接成功，设置socket超时时间为initLimit * tickTime ，然后通过leader.nodelay这个设置来判断是否开启Nagle算法，默认是开启的
4.创建一个LearnerHandler实例来处理与Learner的消息交互并启动它
LernerHandler的run()方法：
public void run() {
try {
learnerMaster.addLearnerHandler(this);
tickOfNextAckDeadline = learnerMaster.getTickOfInitialAckDeadline();
ia = BinaryInputArchive.getArchive(bufferedInput);
bufferedOutput = new BufferedOutputStream(sock.getOutputStream());
oa = BinaryOutputArchive.getArchive(bufferedOutput);
QuorumPacket qp = new QuorumPacket();
ia.readRecord(qp, "packet");
if(qp.getType() != Leader.FOLLOWERINFO && qp.getType() != Leader.OBSERVERINFO){
LOG.error("First packet " + qp.toString()
+ " is not FOLLOWERINFO or OBSERVERINFO!");
return;
}
if (learnerMaster instanceof ObserverMaster && qp.getType() != Leader.OBSERVERINFO) {
throw new IOException("Non observer attempting to connect to ObserverMaster. type = " + qp.getType());
}
byte learnerInfoData[] = qp.getData();
if (learnerInfoData != null) {
ByteBuffer bbsid = ByteBuffer.wrap(learnerInfoData);
if (learnerInfoData.length >= 8) {
this.sid = bbsid.getLong();
}
if (learnerInfoData.length >= 12) {
this.version = bbsid.getInt(); // protocolVersion
}
if (learnerInfoData.length >= 20) {
long configVersion = bbsid.getLong();
if (configVersion > learnerMaster.getQuorumVerifierVersion()) {
throw new IOException("Follower is ahead of the leader (has a later activated configuration)");
}
}
} else {
this.sid = learnerMaster.getAndDecrementFollowerCounter();
}
String followerInfo = learnerMaster.getPeerInfo(this.sid);
if (followerInfo.isEmpty()) {
LOG.info("Follower sid: " + this.sid + " not in the current config "
+ Long.toHexString(learnerMaster.getQuorumVerifierVersion()));
} else {
LOG.info("Follower sid: " + this.sid + " : info : " + followerInfo);
}
if (qp.getType() == Leader.OBSERVERINFO) {
learnerType = LearnerType.OBSERVER;
}
learnerMaster.registerLearnerHandlerBean(this, sock);
long lastAcceptedEpoch = ZxidUtils.getEpochFromZxid(qp.getZxid());
long peerLastZxid;
StateSummary ss = null;
long zxid = qp.getZxid();
long newEpoch = learnerMaster.getEpochToPropose(this.getSid(), lastAcceptedEpoch);
long newLeaderZxid = ZxidUtils.makeZxid(newEpoch, 0);
if (this.getVersion() < 0x10000) {
// we are going to have to extrapolate the epoch information
long epoch = ZxidUtils.getEpochFromZxid(zxid);
ss = new StateSummary(epoch, zxid);
// fake the message
learnerMaster.waitForEpochAck(this.getSid(), ss);
} else {
byte ver[] = new byte[4];
ByteBuffer.wrap(ver).putInt(0x10000);
QuorumPacket newEpochPacket = new QuorumPacket(Leader.LEADERINFO, newLeaderZxid, ver, null);
oa.writeRecord(newEpochPacket, "packet");
bufferedOutput.flush();
QuorumPacket ackEpochPacket = new QuorumPacket();
ia.readRecord(ackEpochPacket, "packet");
if (ackEpochPacket.getType() != Leader.ACKEPOCH) {
LOG.error(ackEpochPacket.toString()
+ " is not ACKEPOCH");
return;
}
ByteBuffer bbepoch = ByteBuffer.wrap(ackEpochPacket.getData());
ss = new StateSummary(bbepoch.getInt(), ackEpochPacket.getZxid());
learnerMaster.waitForEpochAck(this.getSid(), ss);
}
peerLastZxid = ss.getLastZxid();
// Take any necessary action if we need to send TRUNC or DIFF
// startForwarding() will be called in all cases
boolean needSnap = syncFollower(peerLastZxid, learnerMaster);
/* if we are not truncating or sending a diff just send a snapshot */
if (needSnap) {
boolean exemptFromThrottle = getLearnerType() != LearnerType.OBSERVER;
LearnerSnapshot snapshot =
learnerMaster.getLearnerSnapshotThrottler().beginSnapshot(exemptFromThrottle);
try {
long zxidToSend = learnerMaster.getZKDatabase().getDataTreeLastProcessedZxid();
oa.writeRecord(new QuorumPacket(Leader.SNAP, zxidToSend, null, null), "packet");
bufferedOutput.flush();
LOG.info("Sending snapshot last zxid of peer is 0x{}, zxid of leader is 0x{}, "
+ "send zxid of db as 0x{}, {} concurrent snapshots, "
+ "snapshot was {} from throttle",
Long.toHexString(peerLastZxid),
Long.toHexString(leaderLastZxid),
Long.toHexString(zxidToSend),
snapshot.getConcurrentSnapshotNumber(),
snapshot.isEssential() ? "exempt" : "not exempt");
// Dump data to peer
learnerMaster.getZKDatabase().serializeSnapshot(oa);
oa.writeString("BenWasHere", "signature");
bufferedOutput.flush();
} finally {
snapshot.close();
ServerMetrics.getMetrics().SNAP_COUNT.add(1);
}
}
else {
ServerMetrics.getMetrics().DIFF_COUNT.add(1);
}
LOG.debug("Sending NEWLEADER message to " + sid);
// the version of this quorumVerifier will be set by leader.lead() in case
// the leader is just being established. waitForEpochAck makes sure that readyToStart is true if
// we got here, so the version was set
if (getVersion() < 0x10000) {
QuorumPacket newLeaderQP = new QuorumPacket(Leader.NEWLEADER,
newLeaderZxid, null, null);
oa.writeRecord(newLeaderQP, "packet");
} else {
QuorumPacket newLeaderQP = new QuorumPacket(Leader.NEWLEADER,
newLeaderZxid, learnerMaster.getQuorumVerifierBytes(), null);
queuedPackets.add(newLeaderQP);
}
bufferedOutput.flush();
// Start thread that blast packets in the queue to learner
startSendingPackets();
/*
* Have to wait for the first ACK, wait until
* the learnerMaster is ready, and only then we can
* start processing messages.
*/
qp = new QuorumPacket();
ia.readRecord(qp, "packet");
if(qp.getType() != Leader.ACK){
LOG.error("Next packet was supposed to be an ACK,"
+ " but received packet: {}", packetToString(qp));
return;
}
if(LOG.isDebugEnabled()){
LOG.debug("Received NEWLEADER-ACK message from " + sid);
}
learnerMaster.waitForNewLeaderAck(getSid(), qp.getZxid());
syncLimitCheck.start();
// now that the ack has been processed expect the syncLimit
sock.setSoTimeout(learnerMaster.syncTimeout());
/*
* Wait until learnerMaster starts up
*/
learnerMaster.waitForStartup();
// Mutation packets will be queued during the serialize,
// so we need to mark when the peer can actually start
// using the data
//
LOG.debug("Sending UPTODATE message to " + sid);
queuedPackets.add(new QuorumPacket(Leader.UPTODATE, -1, null, null));
while (true) {
qp = new QuorumPacket();
ia.readRecord(qp, "packet");
long traceMask = ZooTrace.SERVER_PACKET_TRACE_MASK;
if (qp.getType() == Leader.PING) {
traceMask = ZooTrace.SERVER_PING_TRACE_MASK;
}
if (LOG.isTraceEnabled()) {
ZooTrace.logQuorumPacket(LOG, traceMask, 'i', qp);
}
tickOfNextAckDeadline = learnerMaster.getTickOfNextAckDeadline();
packetsReceived.incrementAndGet();
ByteBuffer bb;
long sessionId;
int cxid;
int type;
switch (qp.getType()) {
case Leader.ACK:
if (this.learnerType == LearnerType.OBSERVER) {
if (LOG.isDebugEnabled()) {
LOG.debug("Received ACK from Observer " + this.sid);
}
}
syncLimitCheck.updateAck(qp.getZxid());
learnerMaster.processAck(this.sid, qp.getZxid(), sock.getLocalSocketAddress());
break;
case Leader.PING:
// Process the touches
ByteArrayInputStream bis = new ByteArrayInputStream(qp
.getData());
DataInputStream dis = new DataInputStream(bis);
while (dis.available() > 0) {
long sess = dis.readLong();
int to = dis.readInt();
learnerMaster.touch(sess, to);
}
break;
case Leader.REVALIDATE:
ServerMetrics.getMetrics().REVALIDATE_COUNT.add(1);
learnerMaster.revalidateSession(qp, this);
break;
case Leader.REQUEST:
bb = ByteBuffer.wrap(qp.getData());
sessionId = bb.getLong();
cxid = bb.getInt();
type = bb.getInt();
bb = bb.slice();
Request si;
if(type == OpCode.sync){
si = new LearnerSyncRequest(this, sessionId, cxid, type, bb, qp.getAuthinfo());
} else {
si = new Request(null, sessionId, cxid, type, bb, qp.getAuthinfo());
}
si.setOwner(this);
learnerMaster.submitLearnerRequest(si);
requestsReceived.incrementAndGet();
break;
default:
LOG.warn("unexpected quorum packet, type: {}", packetToString(qp));
break;
}
}
} catch (IOException e) {
if (sock != null && !sock.isClosed()) {
LOG.error("Unexpected exception causing shutdown while sock "
+ "still open", e);
//close the socket to make sure the
//other side can see it being close
try {
sock.close();
} catch(IOException ie) {
// do nothing
}
}
} catch (InterruptedException e) {
LOG.error("Unexpected exception causing shutdown", e);
} catch (SnapshotThrottleException e) {
LOG.error("too many concurrent snapshots: " + e);
} finally {
LOG.warn("******* GOODBYE {} ********", getRemoteAddress());
shutdown();
}
}
这段代码比较长，分析如下：
- 1.将这个LeanerHandler实例添加到Leader的LeanerHandler列表learners中
- 2.获取下一个ACK截止时间，计算方法为：当前tick+initLimit+syncLimit（上节讲到Leader在一个tickTime周期内会跟Learner进行两次PING的交互，然后这个tick就会自增1）
- 3.分别获取自定义jute协议中的BinaryInputArchive和BinaryOutputArchive以用作数据读写
- 4.读取Learner发来的数据包，判断是否是FOLLOWERINFO或者OBSERVERINFO，如果都不是，那么直接退出run()方法，因为Learner连接上Leader发送的第一个数据包必须是FOLLOWERINFO或者OBSERVERINFO
- 5.判断当前learnerMaster是否是ObserverMaster（learnerMaster分两个:Leader和ObserverMaster）
- 6.如果数据包的data字段存在的话，那么解析这个数据包的data字段，分别获取serverId、版本号version和投票验证器版本configVersion，接下来对比Leader和Learner的投票验证器版本；如果data字段不存在，则以-1再递减的形式赋予serverId
- 7.获取followerInfo以及确定learnerType，接下来就为LearnerHandler注册JMX服务
- 8.通过数据包传来的zxid获取到最后一次通过事务投票的epoch-lastAcceptedEpoch，然后调用getEpochToPropose(long sid, long lastAcceptedEpoch)方法获取当前集群的领导纪元并计算出集群Leader的zxid（上一章节有讲到getEpochToPropose这个方法，执行方其实只有两类即LearnerMaster和LearnerHandler，每个执行这个方法的线程都会执行wait()陷入等待，等待周期为initTime * tickTime，直到投票验证器通过了获取集群领导纪元的提案后退出方法栈执行后面的流程）
- 9.判断协议版本是否是0x10000，最新版本的是0x10000，然后发送LEADERINFO数据包给Learner并等待接收Learner发送的ACKEPOCH数据包，然后调用waitForEpochAck(long sid, StateSummary ss)方法等待initLimit * tickTime时间周期内领导纪元的ack（可以跟第十章节相互印证）；老版本跟新版本的区别是不会发送LEADERINFO信息给Learner
- 10.接下来会进入数据同步流程，具体的下期再讲
- 11.最新版本将NEWLEADER数据包缓存到阻塞队列queuedPackets中以等待稍后的发送，老版本直接发送NEWLEADER数据包给Learner；接下来启动一个发送缓存数据包的线程，这个线程将循环从阻塞队列queuedPackets中获取缓存的数据包并将它发送给相应的Learner；然后等待接收Learner对NEWLEADER消息发送的ACK数据包，接收到之后将会调用waitForNewLeaderAck方法等待initLimit * tickTime时间周期内NEWLEADER的ACK（可以跟第十章节相互印证）
- 12.启动同步检查器syncLimitCheck，设置Socket超时时间为syncLimit * tickTime，然后等待learnerMaster即Leader或者ObserverMaster的Zookeeper服务启动，接下来再缓存UPTODATE数据包到阻塞队列queuedPackets中等待稍后的发送
- 13.循环处理Learner发来的数据包，首先读取数据包，然后设置新的下一个ACK截止时间tickOfNextAckDeadline并统计数据包接收数量，针对不同类型的数据包处理方式也不尽相同，如下：
- ACK：这个是针对事务请求的提案，表示Follower同意当前事务请求的提案，首先会更新同步检查器syncLimitCheck中提案的处理时间，然后将会由Leader进行事务处理（注意：有两个特殊的ACK，其一是LearnerMaster通过LearnerHandler发送UPTODATE数据包之后Learner会发送一个ACK，其二是LearnerMaster通过LearnerHandler发送NEWLEADER数据包之后Learner会发送一个ACK，但是这两个ACK都不会被当成事务请求处理，至于为什么不处理等下期结合其他章节再说）
- PING：这个是处理LearnerMaster主动发送PING数据包给Learner之后，Learner回复的数据包，数据包中的data字段存储的是会话信息，然后将会由LearnerMaster延长会话过期时间
- REVALIDATE：这个是用于重新验证并激活会话的
- REQUEST：这个是处理Learner转发的事务请求以及sync请求，因为Learner无权处理事务请求，应该交由Leader处理
数据包发送线程
protected void startSendingPackets() {
if (!sendingThreadStarted) {
// Start sending packets
new Thread() {
public void run() {
Thread.currentThread().setName(
"Sender-" + sock.getRemoteSocketAddress());
try {
sendPackets();
} catch (InterruptedException e) {
LOG.warn("Unexpected interruption " + e.getMessage());
}
}
}.start();
sendingThreadStarted = true;
} else {
LOG.error("Attempting to start sending thread after it already started");
}
}
private void sendPackets() throws InterruptedException {
long traceMask = ZooTrace.SERVER_PACKET_TRACE_MASK;
while (true) {
try {
QuorumPacket p;
p = queuedPackets.poll();
if (p == null) {
bufferedOutput.flush();
p = queuedPackets.take();
}
ServerMetrics.getMetrics().LEARNER_HANDLER_QP_SIZE.add(Long.toString(this.sid), queuedPackets.size());
if (p instanceof MarkerQuorumPacket) {
MarkerQuorumPacket m = (MarkerQuorumPacket)p;
ServerMetrics.getMetrics().LEARNER_HANDLER_QP_TIME.add(
Long.toString(this.sid),
(System.nanoTime() - m.time) / 1000000L);
continue;
}
queuedPacketsSize.addAndGet(-packetSize(p));
if (p == proposalOfDeath) {
// Packet of death!
break;
}
if (p.getType() == Leader.PING) {
traceMask = ZooTrace.SERVER_PING_TRACE_MASK;
}
if (p.getType() == Leader.PROPOSAL) {
syncLimitCheck.updateProposal(p.getZxid(), System.nanoTime());
}
if (LOG.isTraceEnabled()) {
ZooTrace.logQuorumPacket(LOG, traceMask, 'o', p);
}
// Log the zxid of the last request, if it is a valid zxid.
if (p.getZxid() > 0) {
lastZxid = p.getZxid();
}
oa.writeRecord(p, "packet");
packetsSent.incrementAndGet();
} catch (IOException e) {
if (!sock.isClosed()) {
LOG.warn("Unexpected exception at " + this, e);
try {
// this will cause everything to shutdown on
// this learner handler and will help notify
// the learner/observer instantaneously
sock.close();
} catch(IOException ie) {
LOG.warn("Error closing socket for handler " + this, ie);
}
}
break;
}
}
}
分析如下：
- 1.启动一个线程专门用来发送缓存在queuedPackets队列中的数据包
- 2.循环从queuedPackets这个队列中获取数据包，首先会进行直接获取，如果获取不到则刷新缓冲区将数据发送出去并阻塞直到获取到新的数据包为止
- 3.进行指标统计，如果是事务请求还需更新同步检查器syncLimitCheck
- 4.重置lastZxid并将数据包写到缓冲区，然后统计已发送的数据包
同步检查器syncLimitCheck
private class SyncLimitCheck {
private boolean started = false;
private long currentZxid = 0;
private long currentTime = 0;
private long nextZxid = 0;
private long nextTime = 0;
public synchronized void start() {
started = true;
}
public synchronized void updateProposal(long zxid, long time) {
if (!started) {
return;
}
if (currentTime == 0) {
currentTime = time;
currentZxid = zxid;
} else {
nextTime = time;
nextZxid = zxid;
}
}
public synchronized void updateAck(long zxid) {
if (currentZxid == zxid) {
currentTime = nextTime;
currentZxid = nextZxid;
nextTime = 0;
nextZxid = 0;
} else if (nextZxid == zxid) {
LOG.warn("ACK for " + zxid + " received before ACK for " + currentZxid + "!!!!");
nextTime = 0;
nextZxid = 0;
}
}
public synchronized boolean check(long time) {
if (currentTime == 0) {
return true;
} else {
long msDelay = (time - currentTime) / 1000000;
return (msDelay < learnerMaster.syncTimeout());
}
}
}
- updateProposal：LearnerHandler发送提案时，检查是否已启动以及当前已发送ACK的提案的时间currentTime，如果currentTime等于0，则将这个提案设置为当前提案，否则设置为下一提案
- updateAck：LearnerHandler接收到Learner发送的ACK时，检查当前提案是否是刚刚发送ACK的提案，如果是的话更新当前提案的时间为之前调用updateProposal方法设置的nextTime 和 nextZxid并重置nextTime 和 nextZxid为0，如果不是的话说明请求处理乱序了，也重置nextTime 和 nextZxid为0
- check：LearnerHandler发送PING数据包给Learner之前会进行提案处理时间的检查，检查逻辑就是判断currentTime是否等于0以及当前时间减去currentTime是否小于同步超时时间syncLimit * tickTime；假如一个提案一直未提交，直到发送PING数据包时发现超时将关闭当前LearnerHandler（注意：一个tickTime周期内发送两次PING数据包）