Kafka消费者源码解析之三ConsumerNetworkClient
ConsumerNetworkClient解析
- 开篇
- ConsumerNetworkClient 概述
- 成员变量
- 内部类和内部接口
- RequestFutureCompletionHandler 内部类
- UnsentRequests 内部类
- PollCondition 内部接口
- 主要方法
- maybeTriggerWakeup 方法
- poll 方法
- 小结
开篇
通过消费者第一篇KafkaConsumer解析，我们可以知道，client主要做了两件事情：
- maybeTriggerWakeup() 安全的唤醒消费者客户端
- poll() 轮询任何网络IO
ConsumerNetworkClient 概述
根据名字我们也能看出，此类是消费者访问网路客户端。
官方定义：
- 更高级别的消费者访问网络层。
- 该类是线程安全的，但不提供响应回调的同步。这保证在调用锁时不会持有锁。
说白了就是处理网络IO的类。
成员变量
private static final int MAX_POLL_TIMEOUT_MS = 5000; // 允许的最大超时时间
// the mutable state of this class is protected by the object's monitor (excluding the wakeup
// flag and the request completion queue below).
private final Logger log;
private final KafkaClient client; // kafka客户端
private final UnsentRequests unsent = new UnsentRequests();
private final Metadata metadata; // 元数据类
private final Time time;
private final long retryBackoffMs;
private final int maxPollTimeoutMs;
private final int requestTimeoutMs;
private final AtomicBoolean wakeupDisabled = new AtomicBoolean(); // 执行不可中断方法的标志
// We do not need high throughput, so use a fair lock to try to avoid starvation
private final ReentrantLock lock = new ReentrantLock(true);
// when requests complete, they are transferred to this queue prior to invocation. The purpose
// is to avoid invoking them while holding this object's monitor which can open the door for deadlocks.
// 当请求完成时，它们在调用之前被传输到这个队列。这样做的目的是避免在持有此对象的监视器时调用它们，因为该对象的监视器会打开死锁。
private final ConcurrentLinkedQueue<RequestFutureCompletionHandler> pendingCompletion = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Node> pendingDisconnects = new ConcurrentLinkedQueue<>();
// this flag allows the client to be safely woken up without waiting on the lock above. It is
// atomic to avoid the need to acquire the lock above in order to enable it concurrently.
private final AtomicBoolean wakeup = new AtomicBoolean(false);
成员变量不多，也挺简单，接下来看下内部类和内部接口
内部类和内部接口
它拥有两个内部类和一个内部接口。第一次见到内部接口，挺期待呢。
RequestFutureCompletionHandler 内部类
此类主要作用就是记录请求过程中的异常和请求完成之后的结果。
private class RequestFutureCompletionHandler implements RequestCompletionHandler {
private final RequestFuture<ClientResponse> future; // 异步请求结果
private ClientResponse response; // 客户端响应结果
private RuntimeException e; // 运行时异常
private RequestFutureCompletionHandler() {
this.future = new RequestFuture<>();
}
public void fireCompletion() {
//判断有无运行时异常
if (e != null) {
future.raise(e);
// 判断有无身份认证异常
} else if (response.authenticationException() != null) {
future.raise(response.authenticationException());
// 判断有无断开连接异常
} else if (response.wasDisconnected()) {
log.debug("Cancelled request with header {} due to node {} being disconnected",
response.requestHeader(), response.destination());
future.raise(DisconnectException.INSTANCE);
// 判断有无不支持版本异常
} else if (response.versionMismatch() != null) {
future.raise(response.versionMismatch());
} else {
// 完成请求
future.complete(response);
}
}
// 失败，将运行时异常传入
public void onFailure(RuntimeException e) {
this.e = e;
pendingCompletion.add(this);
}
@Override
// 成功，将响应结果传入
public void onComplete(ClientResponse response) {
this.response = response;
pendingCompletion.add(this);
}
}
UnsentRequests 内部类
一个线程安全的助手类，用于保存尚未发送的每个节点的请求。
此类比较简单，主要就是维护一个 节点与请求并发队列的 map集合，并提供了足够全面的操作该集合的方法。
private final static class UnsentRequests {
private final ConcurrentMap<Node, ConcurrentLinkedQueue<ClientRequest>> unsent; // 节点与请求并发队列的线程安全/高效的map集合
private UnsentRequests() {
unsent = new ConcurrentHashMap<>();
}
// 将节点和请求加入map集合中
public void put(Node node, ClientRequest request) {
// the lock protects the put from a concurrent removal of the queue for the node
synchronized (unsent) {
// 通过节点找到对应的请求队列
ConcurrentLinkedQueue<ClientRequest> requests = unsent.get(node);
if (requests == null) {
requests = new ConcurrentLinkedQueue<>();
unsent.put(node, requests);
}
// 将请求加入到并发队列中
requests.add(request);
}
}
// 统计某个节点上的请求总数
public int requestCount(Node node) {
ConcurrentLinkedQueue<ClientRequest> requests = unsent.get(node);
return requests == null ? 0 : requests.size();
}
// 统计当前 map集合里所有节点的请求总数
public int requestCount() {
int total = 0;
for (ConcurrentLinkedQueue<ClientRequest> requests : unsent.values())
total += requests.size();
return total;
}
// 判断某个节点是否有请求
public boolean hasRequests(Node node) {
ConcurrentLinkedQueue<ClientRequest> requests = unsent.get(node);
return requests != null && !requests.isEmpty();
}
// 判断当前 map集合里所有节点的是否有请求
public boolean hasRequests() {
for (ConcurrentLinkedQueue<ClientRequest> requests : unsent.values())
if (!requests.isEmpty())
return true;
return false;
}
// 清空map集合里，所有队列中过期的请求
private Collection<ClientRequest> removeExpiredRequests(long now) {
List<ClientRequest> expiredRequests = new ArrayList<>();
for (ConcurrentLinkedQueue<ClientRequest> requests : unsent.values()) {
Iterator<ClientRequest> requestIterator = requests.iterator();
while (requestIterator.hasNext()) {
ClientRequest request = requestIterator.next();
long elapsedMs = Math.max(0, now - request.createdTimeMs());
if (elapsedMs > request.requestTimeoutMs()) {
expiredRequests.add(request);
requestIterator.remove();
} else
break;
}
}
return expiredRequests;
}
// 清空map 集合
public void clean() {
// the lock protects removal from a concurrent put which could otherwise mutate the
// queue after it has been removed from the map
synchronized (unsent) {
Iterator<ConcurrentLinkedQueue<ClientRequest>> iterator = unsent.values().iterator();
while (iterator.hasNext()) {
ConcurrentLinkedQueue<ClientRequest> requests = iterator.next();
if (requests.isEmpty())
iterator.remove();
}
}
}
// 移除map集合里的某个节点
public Collection<ClientRequest> remove(Node node) {
// the lock protects removal from a concurrent put which could otherwise mutate the
// queue after it has been removed from the map
synchronized (unsent) {
ConcurrentLinkedQueue<ClientRequest> requests = unsent.remove(node);
return requests == null ? Collections.<ClientRequest>emptyList() : requests;
}
}
// 获取map集合中 某个节点的请求集合
public Iterator<ClientRequest> requestIterator(Node node) {
ConcurrentLinkedQueue<ClientRequest> requests = unsent.get(node);
return requests == null ? Collections.<ClientRequest>emptyIterator() : requests.iterator();
}
// 获取map集合中 所有节点集合
public Collection<Node> nodes() {
return unsent.keySet();
}
}
PollCondition 内部接口
返回调用者是否仍然在等待IO事件。相当于一个标志，如果在调用shouldBlock()之后满足了预期的完成条件(由于触发了完成处理程序)，客户机将被唤醒。
public interface PollCondition {
/**
* Return whether the caller is still awaiting an IO event.
* @return true if so, false otherwise.
*/
boolean shouldBlock();
}
主要方法
开篇的时候就简单介绍了两个方法，下面详细介绍下
maybeTriggerWakeup 方法
其实这个方法在消费者第一篇KafkaConsumer解析里面就已经介绍过了，这个方法就是为了安全的唤醒消费者客户端。注意：并不是在这里唤醒，这里是判断是否有唤醒的风险。
这里维护了两个原子标志（线程安全）：
- wakeupDisabled： 线程在执行不可中断的方法
- wakeup： 线程中断请求
- get() 方法是判断值是否为0，非0返回 True
当wakeupDisabled 为 0 并且 wakeup 是 1，即线程没有在执行不可中断的方法，并收到了中断请求，则把 wakeup置为 0 并抛出异常，中断该线程。
public void maybeTriggerWakeup() {
if (!wakeupDisabled.get() && wakeup.get()) {
log.debug("Raising WakeupException in response to user wakeup");
wakeup.set(false);
throw new WakeupException();
}
}
poll 方法
此方法是核心方法，处理任何网络IO。
public void poll(Timer timer, PollCondition pollCondition, boolean disableWakeup) {
// there may be handlers which need to be invoked if we woke up the previous call to poll
// 如果我们唤醒前一个轮询调用，可能需要调用一些处理程序
firePendingCompletedRequests();
// 添加非公平锁
lock.lock();
try {
// Handle async disconnects prior to attempting any sends
// 在尝试任何发送之前处理异步断开，并存入已完成请求的队列中
handlePendingDisconnects();
// send all the requests we can send now
// 发送所有我们现在可以发送的请求
long pollDelayMs = trySend(timer.currentTimeMs());
// check whether the poll is still needed by the caller. Note that if the expected completion
// condition becomes satisfied after the call to shouldBlock() (because of a fired completion
// handler), the client will be woken up.
// 判断已完成的请求是否为空
// 判断内部接口是否为空
// 判断内部接口的实现方法
if (pendingCompletion.isEmpty() && (pollCondition == null || pollCondition.shouldBlock())) {
// if there are no requests in flight, do not block longer than the retry backoff
long pollTimeout = Math.min(timer.remainingMs(), pollDelayMs);
if (client.inFlightRequestCount() == 0)
// 如果在飞行中没有请求，则不要阻塞超过重试回退的时间
pollTimeout = Math.min(pollTimeout, retryBackoffMs);
// 延时发送请求
client.poll(pollTimeout, timer.currentTimeMs());
} else {
// 立即发送请求
client.poll(0, timer.currentTimeMs());
}
timer.update();
// handle any disconnects by failing the active requests. note that disconnects must
// be checked immediately following poll since any subsequent call to client.ready()
// will reset the disconnect status
// 检查未发送请求的连接是否已断开;如果有，则完成相应的future并在ClientResponse中设置断开连接标志
checkDisconnects(timer.currentTimeMs());
// 判断是否禁用触发唤醒
if (!disableWakeup) {
// trigger wakeups after checking for disconnects so that the callbacks will be ready
// to be fired on the next call to poll()
// 安全的触发唤醒，以便在下一次调用poll()时触发回调
maybeTriggerWakeup();
}
// throw InterruptException if this thread is interrupted
// 如果线程被中断，抛出异常
maybeThrowInterruptException();
// try again to send requests since buffer space may have been
// cleared or a connect finished in the poll
// 再次尝试发送请求，因为缓冲区空间可能已被清除，或者在轮询中连接已完成
trySend(timer.currentTimeMs());
// fail requests that couldn't be sent if they have expired
// 将无法发送的失败请求存入 已完成请求队列中
failExpiredRequests(timer.currentTimeMs());
// clean unsent requests collection to keep the map from growing indefinitely
// 清除未发送请求集合，以防止映射无限期增长
unsent.clean();
} finally {
// 释放公平锁
lock.unlock();
}
// called without the lock to avoid deadlock potential if handlers need to acquire locks
// 在没有锁的情况下调用，以避免在处理程序需要获取锁时发生死锁
firePendingCompletedRequests();
}
小结
到此，消费者访问网路客户端这个类就大概介绍完了，我们了解了它如何安全的被唤醒，如何处理各种情况的请求。如有不对之处，还望指正。
接下来我打算总结下消费者总体流程，敬请期待。