RocketMQ源码解析之NameServer请求处理
阅读须知
- 文章中使用/* */注释的方法会做深入分析
正文
NameServer的请求处理流程，我们主要聚焦在构建netty时添加的一些handler，首先我们来看处理握手的handler：
NettyRemotingServer.HandshakeHandler：
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception {
// 标记当前位置，以便我们可以查看第一个字节以确定内容是否以TLS握手开始
msg.markReaderIndex();
byte b = msg.getByte(0);
if (b == HANDSHAKE_MAGIC_CODE) {
switch (tlsMode) {
// 服务器禁用SSL，但客户端打算建立SSL连接的场景
case DISABLED:
ctx.close();
log.warn("Clients intend to establish a SSL connection while this server is running in SSL disabled mode");
break;
case PERMISSIVE:
case ENFORCING:
if (null != sslContext) {
ctx.pipeline()
// 添加netty提供的SslHandler以提供对SSL协议的支持
.addAfter(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, TLS_HANDLER_NAME, sslContext.newHandler(ctx.channel().alloc()))
// 文件区域直接传输到channel，支持零拷贝
.addAfter(defaultEventExecutorGroup, TLS_HANDLER_NAME, FILE_REGION_ENCODER_NAME, new FileRegionEncoder());
log.info("Handlers prepended to channel pipeline to establish SSL connection");
} else {
// 尝试建立SSL连接但sslContext为null的处理
ctx.close();
log.error("Trying to establish a SSL connection but sslContext is null");
}
break;
default:
log.warn("Unknown TLS mode");
break;
}
} else if (tlsMode == TlsMode.ENFORCING) {
ctx.close();
log.warn("Clients intend to establish an insecure connection while this server is running in SSL enforcing mode");
}
// 重置reader index，以便握手可以正常进行
msg.resetReaderIndex();
try {
// 移除掉此handler，握手时处理一次就可以
ctx.pipeline().remove(this);
} catch (NoSuchElementException e) {
log.error("Error while removing HandshakeHandler", e);
}
// 将此消息移交给下一个handler
ctx.fireChannelRead(msg.retain());
}
另外一个NettyConnectManageHandler的主要作用是根据channel的状态来触发不同的事件，事件的类型定义在NettyEventType这个枚举中，分为CONNECT、CLOSE、IDLE、EXCEPTION四个事件类型，而这些事件的监听器是在构造NamesrvController时创建的BrokerHousekeepingService，BrokerHousekeepingService的作用是在接收到CLOSE、IDLE、EXCEPTION三个事件时调用RouteInfoManager的onChannelDestroy方法释放一些资源，这个方法我们在分析NameServer启动和停止流程时分析过，这里不再重复。
真正处理请求的handler为NettyRemotingServer.NettyServerHandler：
protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
processMessageReceived(ctx, msg);
}
public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception {
final RemotingCommand cmd = msg;
if (cmd != null) {
switch (cmd.getType()) {
case REQUEST_COMMAND:
/* 处理请求命令 */
processRequestCommand(ctx, cmd);
break;
case RESPONSE_COMMAND:
processResponseCommand(ctx, cmd);
break;
default:
break;
}
}
}
NettyRemotingAbstract：
public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) {
// 根据request code从processor映射中获取processor处理器
final Pair<NettyRequestProcessor, ExecutorService> matched = this.processorTable.get(cmd.getCode());
// 如果没有匹配到则使用默认的defaultRequestProcessor，defaultRequestProcessor我们在NameServer启动流程中提及过
final Pair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessor : matched;
final int opaque = cmd.getOpaque();
if (pair != null) {
Runnable run = new Runnable() {
@Override
public void run() {
try {
// 请求处理之前的扩展钩子调用，我们可以实现RPCHook来自定义处理逻辑，NameServer默认没有实现RPCHook
doBeforeRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd);
/* 请求处理 */
final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd);
// 请求处理之后的扩展钩子调用
doAfterRpcHooks(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response);
// 如果不是单向请求需要发送响应
if (!cmd.isOnewayRPC()) {
if (response != null) {
response.setOpaque(opaque);
// 设置command为响应类型
response.markResponseType();
try {
ctx.writeAndFlush(response);
} catch (Throwable e) {
log.error("process request over, but response failed", e);
log.error(cmd.toString());
log.error(response.toString());
}
} else {
}
}
} catch (Throwable e) {
log.error("process request exception", e);
log.error(cmd.toString());
if (!cmd.isOnewayRPC()) {
final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR,
RemotingHelper.exceptionSimpleDesc(e));
response.setOpaque(opaque);
ctx.writeAndFlush(response);
}
}
}
};
// 判断processor是否拒接请求
if (pair.getObject1().rejectRequest()) {
// 响应系统繁忙
final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,
"[REJECTREQUEST]system busy, start flow control for a while");
response.setOpaque(opaque);
ctx.writeAndFlush(response);
return;
}
try {
// 构建请求处理任务
final RequestTask requestTask = new RequestTask(run, ctx.channel(), cmd);
// 使用processor对应的线程池来处理任务
pair.getObject2().submit(requestTask);
} catch (RejectedExecutionException e) {
if ((System.currentTimeMillis() % 10000) == 0) {
log.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel())
+ ", too many requests and system thread pool busy, RejectedExecutionException "
+ pair.getObject2().toString()
+ " request code: " + cmd.getCode());
}
// 如果处理的线程池因为饱和策略抛出了RejectedExecutionException，非单向的请求响应系统繁忙
if (!cmd.isOnewayRPC()) {
final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,
"[OVERLOAD]system busy, start flow control for a while");
response.setOpaque(opaque);
ctx.writeAndFlush(response);
}
}
} else {
String error = " request type " + cmd.getCode() + " not supported";
// 如果没有获取到请求处理器，响应request code不支持
final RemotingCommand response =
RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error);
response.setOpaque(opaque);
ctx.writeAndFlush(response);
log.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error);
}
}
这里的请求处理器映射也就是processorTable默认是空的，所以请求都会通过默认的请求处理器defaultRequestProcessor来处理，在NameServer启动流程中，我们看到defaultRequestProcessor的注册是在NamesrvController.initialize方法中完成的，默认请求处理器的类型是DefaultRequestProcessor，我们来看DefaultRequestProcessor处理请求的过程：
public RemotingCommand processRequest(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
if (ctx != null) {
log.debug("receive request, {} {} {}",
request.getCode(),
RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
request);
}
switch (request.getCode()) {
case RequestCode.PUT_KV_CONFIG:
// 设置kv配置
return this.putKVConfig(ctx, request);
case RequestCode.GET_KV_CONFIG:
// 获取kv配置
return this.getKVConfig(ctx, request);
case RequestCode.DELETE_KV_CONFIG:
// 删除kv配置
return this.deleteKVConfig(ctx, request);
case RequestCode.QUERY_DATA_VERSION:
// 从brokerLiveTable中查询data version并判断是否已经改变
return queryBrokerTopicConfig(ctx, request);
case RequestCode.REGISTER_BROKER:
Version brokerVersion = MQVersion.value2Version(request.getVersion());
if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) {
/* 注册broker */
return this.registerBrokerWithFilterServer(ctx, request);
} else {
return this.registerBroker(ctx, request);
}
case RequestCode.UNREGISTER_BROKER:
/* 注销broker */
return this.unregisterBroker(ctx, request);
case RequestCode.GET_ROUTEINTO_BY_TOPIC:
// 根据topic后去路由信息
return this.getRouteInfoByTopic(ctx, request);
case RequestCode.GET_BROKER_CLUSTER_INFO:
// 获取broker的集群信息
return this.getBrokerClusterInfo(ctx, request);
case RequestCode.WIPE_WRITE_PERM_OF_BROKER:
// 擦写broker的持久化标记
return this.wipeWritePermOfBroker(ctx, request);
case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER:
// 获取所有的topic列表
return getAllTopicListFromNameserver(ctx, request);
case RequestCode.DELETE_TOPIC_IN_NAMESRV:
// 删除topic
return deleteTopicInNamesrv(ctx, request);
case RequestCode.GET_KVLIST_BY_NAMESPACE:
// 根据namespace获取kv列表
return this.getKVListByNamespace(ctx, request);
case RequestCode.GET_TOPICS_BY_CLUSTER:
// 获取集群的topic列表
return this.getTopicsByCluster(ctx, request);
case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS:
// 获取系统的topic列表
return this.getSystemTopicListFromNs(ctx, request);
case RequestCode.GET_UNIT_TOPIC_LIST:
// 获取单元topic列表，单元意义是假如我们的集群部署在两个机房中，这时我们就可以设置两个单元来避免跨机房调用
return this.getUnitTopicList(ctx, request);
case RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST:
// 获取子单元topic列表
return this.getHasUnitSubTopicList(ctx, request);
case RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST:
// 获取子单元非单元topic列表
return this.getHasUnitSubUnUnitTopicList(ctx, request);
case RequestCode.UPDATE_NAMESRV_CONFIG:
// 更新name server配置
return this.updateConfig(ctx, request);
case RequestCode.GET_NAMESRV_CONFIG:
// 获取name server配置信息
return this.getConfig(ctx, request);
default:
break;
}
return null;
}
这里很多操作都与RouteInfoManager中维护的各个映射有关，这些映射中大部分我们在NameServer启动流程中都有提及过。这里我们着重来看下broker的注册和注销：
DefaultRequestProcessor：
public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class);
final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader();
final RegisterBrokerRequestHeader requestHeader =
(RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class);
// 检查crc32校验和
if (!checksum(ctx, request, requestHeader)) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("crc32 not match");
return response;
}
RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody();
if (request.getBody() != null) {
try {
// 解码注册broker请求体
registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), requestHeader.isCompressed());
} catch (Exception e) {
throw new RemotingCommandException("Failed to decode RegisterBrokerBody", e);
}
} else {
registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setCounter(new AtomicLong(0));
registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestamp(0);
}
/* 注册broker */
RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker(
requestHeader.getClusterName(),
requestHeader.getBrokerAddr(),
requestHeader.getBrokerName(),
requestHeader.getBrokerId(),
requestHeader.getHaServerAddr(),
registerBrokerBody.getTopicConfigSerializeWrapper(),
registerBrokerBody.getFilterServerList(),
ctx.channel());
responseHeader.setHaServerAddr(result.getHaServerAddr());
responseHeader.setMasterAddr(result.getMasterAddr());
byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG);
response.setBody(jsonValue);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
}
RouteInfoManager：
public RegisterBrokerResult registerBroker(
final String clusterName,
final String brokerAddr,
final String brokerName,
final long brokerId,
final String haServerAddr,
final TopicConfigSerializeWrapper topicConfigWrapper,
final List<String> filterServerList,
final Channel channel) {
RegisterBrokerResult result = new RegisterBrokerResult();
try {
try {
this.lock.writeLock().lockInterruptibly();
Set<String> brokerNames = this.clusterAddrTable.get(clusterName);
// 判断集群和broker的映射关系是否存在并决定是否添加映射
if (null == brokerNames) {
brokerNames = new HashSet<String>();
this.clusterAddrTable.put(clusterName, brokerNames);
}
brokerNames.add(brokerName);
// 标记是否为第一次注册
boolean registerFirst = false;
BrokerData brokerData = this.brokerAddrTable.get(brokerName);
// 判断broker和BrokerData（broker名称、地址等数据）的映射关系是否存在并决定是否添加映射
if (null == brokerData) {
// 如果BrokerData映射为null，说明是第一次注册
registerFirst = true;
brokerData = new BrokerData(clusterName, brokerName, new HashMap<Long, String>());
this.brokerAddrTable.put(brokerName, brokerData);
}
// 向broker的地址列表中添加brokerId和broker地址的关系映射
String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr);
registerFirst = registerFirst || (null == oldAddr);
// 判断是否是master broker（RocketMQ broker采用master/slave结构来保证高可用）
if (null != topicConfigWrapper
&& MixAll.MASTER_ID == brokerId) {
// 如果topic配置改变或者第一次注册，需要创建或更新队列数据
if (this.isBrokerTopicConfigChanged(brokerAddr, topicConfigWrapper.getDataVersion())
|| registerFirst) {
ConcurrentMap<String, TopicConfig> tcTable =
topicConfigWrapper.getTopicConfigTable();
if (tcTable != null) {
for (Map.Entry<String, TopicConfig> entry : tcTable.entrySet()) {
/* 创建或更新队列数据 */
this.createAndUpdateQueueData(brokerName, entry.getValue());
}
}
}
}
// 添加“broker地址 --> 存活的broker信息”映射
BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddr,
new BrokerLiveInfo(
System.currentTimeMillis(),
topicConfigWrapper.getDataVersion(),
channel,
haServerAddr));
if (null == prevBrokerLiveInfo) {
log.info("new broker registered, {} HAServer: {}", brokerAddr, haServerAddr);
}
if (filterServerList != null) {
// 判断过滤器服务列表是否为空来确定在“broker地址 --> 过滤器服务列表”映射中移除或添加映射
if (filterServerList.isEmpty()) {
this.filterServerTable.remove(brokerAddr);
} else {
this.filterServerTable.put(brokerAddr, filterServerList);
}
}
// 不是master broker，也就是slave broker，需要拿到master broker的信息
if (MixAll.MASTER_ID != brokerId) {
String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID);
if (masterAddr != null) {
BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.get(masterAddr);
if (brokerLiveInfo != null) {
result.setHaServerAddr(brokerLiveInfo.getHaServerAddr());
result.setMasterAddr(masterAddr);
}
}
}
} finally {
this.lock.writeLock().unlock();
}
} catch (Exception e) {
log.error("registerBroker Exception", e);
}
return result;
}
RouteInfoManager：
private void createAndUpdateQueueData(final String brokerName, final TopicConfig topicConfig) {
// 根据topic配置信息构建队列数据
QueueData queueData = new QueueData();
queueData.setBrokerName(brokerName);
queueData.setWriteQueueNums(topicConfig.getWriteQueueNums());
queueData.setReadQueueNums(topicConfig.getReadQueueNums());
queueData.setPerm(topicConfig.getPerm());
queueData.setTopicSynFlag(topicConfig.getTopicSysFlag());
// topic名称 --> 队列数据列表
List<QueueData> queueDataList = this.topicQueueTable.get(topicConfig.getTopicName());
if (null == queueDataList) {
queueDataList = new LinkedList<QueueData>();
queueDataList.add(queueData);
// 映射为null添加映射
this.topicQueueTable.put(topicConfig.getTopicName(), queueDataList);
log.info("new topic registered, {} {}", topicConfig.getTopicName(), queueData);
} else {
boolean addNewOne = true;
Iterator<QueueData> it = queueDataList.iterator();
while (it.hasNext()) {
QueueData qd = it.next();
// 遍历找到broker name匹配的QueueData
if (qd.getBrokerName().equals(brokerName)) {
// 相等不做处理，不相等移除
if (qd.equals(queueData)) {
addNewOne = false;
} else {
log.info("topic changed, {} OLD: {} NEW: {}", topicConfig.getTopicName(), qd,
queueData);
it.remove();
}
}
}
// 如果不存在相等的，添加新的QueueData
if (addNewOne) {
queueDataList.add(queueData);
}
}
}
注册broker主要是在RouteInfoManager类中维护的一些映射关系中添加或更新相关数据，所以反过来注销broker就是从这些映射中移除相关数据：
DefaultRequestProcessor：
public RemotingCommand unregisterBroker(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final UnRegisterBrokerRequestHeader requestHeader =
(UnRegisterBrokerRequestHeader) request.decodeCommandCustomHeader(UnRegisterBrokerRequestHeader.class);
/* 注销broker */
this.namesrvController.getRouteInfoManager().unregisterBroker(
requestHeader.getClusterName(),
requestHeader.getBrokerAddr(),
requestHeader.getBrokerName(),
requestHeader.getBrokerId());
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
}
RouteInfoManager：
public void unregisterBroker(
final String clusterName,
final String brokerAddr,
final String brokerName,
final long brokerId) {
try {
try {
this.lock.writeLock().lockInterruptibly();
// 从“broker地址 --> 存活的broker信息”映射中移除指定地址
BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddr);
log.info("unregisterBroker, remove from brokerLiveTable {}, {}",
brokerLiveInfo != null ? "OK" : "Failed",
brokerAddr
);
// 从“broker地址 --> 过滤器服务列表”映射中移除指定地址
this.filterServerTable.remove(brokerAddr);
// 标记是否移除了吧broker
boolean removeBrokerName = false;
BrokerData brokerData = this.brokerAddrTable.get(brokerName);
if (null != brokerData) {
// 从broker地址列表中移除指定brokerId对应的地址
String addr = brokerData.getBrokerAddrs().remove(brokerId);
log.info("unregisterBroker, remove addr from brokerAddrTable {}, {}",
addr != null ? "OK" : "Failed",
brokerAddr
);
// 如果移除给定的地址后地址列表为空了，则需要将broker移除
if (brokerData.getBrokerAddrs().isEmpty()) {
// 从broker地址列表中移除
this.brokerAddrTable.remove(brokerName);
log.info("unregisterBroker, remove name from brokerAddrTable OK, {}",
brokerName
);
// 标记为需要移除的broker
removeBrokerName = true;
}
}
// 判断是否需要移除broker
if (removeBrokerName) {
Set<String> nameSet = this.clusterAddrTable.get(clusterName);
if (nameSet != null) {
// 从集群中移除该broker
boolean removed = nameSet.remove(brokerName);
log.info("unregisterBroker, remove name from clusterAddrTable {}, {}",
removed ? "OK" : "Failed",
brokerName);
// 移除broker后如果集群broker列表已经为空，则需要移除此集群
if (nameSet.isEmpty()) {
this.clusterAddrTable.remove(clusterName);
log.info("unregisterBroker, remove cluster from clusterAddrTable {}",
clusterName
);
}
}
/* 根据broker移除topic */
this.removeTopicByBrokerName(brokerName);
}
} finally {
this.lock.writeLock().unlock();
}
} catch (Exception e) {
log.error("unregisterBroker Exception", e);
}
}
RouteInfoManager：
private void removeTopicByBrokerName(final String brokerName) {
Iterator<Entry<String, List<QueueData>>> itMap = this.topicQueueTable.entrySet().iterator();
while (itMap.hasNext()) {
Entry<String, List<QueueData>> entry = itMap.next();
String topic = entry.getKey();
List<QueueData> queueDataList = entry.getValue();
Iterator<QueueData> it = queueDataList.iterator();
while (it.hasNext()) {
QueueData qd = it.next();
// 匹配需要移除的broker移除队列数据
if (qd.getBrokerName().equals(brokerName)) {
log.info("removeTopicByBrokerName, remove one broker's topic {} {}", topic, qd);
it.remove();
}
}
// 如果移除后队列列表为空，则移除此“topic --> 队列数据列表”映射信息
if (queueDataList.isEmpty()) {
log.info("removeTopicByBrokerName, remove the topic all queue {}", topic);
itMap.remove();
}
}
}
到这里，NameServer请求处理流程就分析完成了。