RocketMQ消息消费三：负载均衡
概述
这篇文章介绍RocketMQ消费消息的负载均衡策略。因为消费消息的方式push&pull实际上均由pull模式实现，所以具体负载均衡是在客户端完成的。
核心类—RebalanceImpl
在Consumer启动的过程中会完成RebalanceService启动，每20s执行一次，最终调用RebalanceImpl来实现。RebalanceImpl为抽象类，子类有RebalancePullImpl和RebalancePushImpl两个，分别对应pull和push模式
源码分析
入口
最终入口为RebalanceImpl.doRebalance。
public void doRebalance(final boolean isOrder) {
Map<String, SubscriptionData> subTable = this.getSubscriptionInner();
if (subTable != null) {
for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) {
final String topic = entry.getKey();
try {
// 对每个topic 调用 rebalanceByTopic
this.rebalanceByTopic(topic, isOrder);
} catch (Throwable e) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("rebalanceByTopic Exception", e);
}
}
}
}
// 清理 负载均衡后，分配到其他消费者的消费队列
this.truncateMessageQueueNotMyTopic();
}
每个topic做负载均衡
private void rebalanceByTopic(final String topic, final boolean isOrder) {
switch (messageModel) {
case BROADCASTING: {
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
if (mqSet != null) {
boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder);
if (changed) {
this.messageQueueChanged(topic, mqSet, mqSet);
log.info("messageQueueChanged {} {} {} {}",
consumerGroup,
topic,
mqSet,
mqSet);
}
} else {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
break;
}
// 主要关注集群模式
case CLUSTERING: {
// 获取 消费队列的列表
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
// 从broker获取消费者列表
List<String> cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup);
if (null == mqSet) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
}
if (null == cidAll) {
log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic);
}
if (mqSet != null && cidAll != null) {
List<MessageQueue> mqAll = new ArrayList<MessageQueue>();
mqAll.addAll(mqSet);
// 排序，保证所有客户端做负载均衡的顺序一致
Collections.sort(mqAll);
Collections.sort(cidAll);
// 指定的负载均衡策略，后面介绍
AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy;
List<MessageQueue> allocateResult = null;
try {
// 执行负载均衡
allocateResult = strategy.allocate(
this.consumerGroup,
this.mQClientFactory.getClientId(),
mqAll,
cidAll);
} catch (Throwable e) {
log.error("AllocateMessageQueueStrategy.allocate Exception. allocateMessageQueueStrategyName={}", strategy.getName(),
e);
return;
}
Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
if (allocateResult != null) {
allocateResultSet.addAll(allocateResult);
}
/** 更新负载均衡后的队列信息
* 主要 1、对原来由本消费者消费，负载均衡后交给其他消费者消费的队列，做停止消费处理
* 2、对原来由其他消费者消费，负载均衡后交给本消费者消费的队列，做开始消费处理
*/
boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder);
if (changed) {
log.info(
"rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, rebalanceResultSize={}, rebalanceResultSet={}",
strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(),
allocateResultSet.size(), allocateResultSet);
this.messageQueueChanged(topic, mqSet, allocateResultSet);
}
}
break;
}
default:
break;
}
}
做负载均衡后处理
// 更新ProcessQueueTable
private boolean updateProcessQueueTableInRebalance(final String topic, final Set<MessageQueue> mqSet,
final boolean isOrder) {
boolean changed = false;
Iterator<Entry<MessageQueue, ProcessQueue>> it = this.processQueueTable.entrySet().iterator();
while (it.hasNext()) {
Entry<MessageQueue, ProcessQueue> next = it.next();
MessageQueue mq = next.getKey();
ProcessQueue pq = next.getValue();
if (mq.getTopic().equals(topic)) {
// 负载均衡后，topic交由其他 消费者消费，将ProcessQueue drop
if (!mqSet.contains(mq)) {
pq.setDropped(true);
if (this.removeUnnecessaryMessageQueue(mq, pq)) {
it.remove();
changed = true;
log.info("doRebalance, {}, remove unnecessary mq, {}", consumerGroup, mq);
}
} else if (pq.isPullExpired()) { // 过期
switch (this.consumeType()) {
case CONSUME_ACTIVELY:
break;
case CONSUME_PASSIVELY:
pq.setDropped(true);
if (this.removeUnnecessaryMessageQueue(mq, pq)) {
it.remove();
changed = true;
log.error("[BUG]doRebalance, {}, remove unnecessary mq, {}, because pull is pause, so try to fixed it",
consumerGroup, mq);
}
break;
default:
break;
}
}
}
}
List<PullRequest> pullRequestList = new ArrayList<PullRequest>();
for (MessageQueue mq : mqSet) {
// 负载均衡后，新分配到 本消费者的队列
if (!this.processQueueTable.containsKey(mq)) {
if (isOrder && !this.lock(mq)) {
log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq);
continue;
}
this.removeDirtyOffset(mq);
ProcessQueue pq = new ProcessQueue();
// 计算offset
long nextOffset = this.computePullFromWhere(mq);
if (nextOffset >= 0) {
// 启动消费
ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq);
if (pre != null) {
log.info("doRebalance, {}, mq already exists, {}", consumerGroup, mq);
} else {
log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq);
PullRequest pullRequest = new PullRequest();
pullRequest.setConsumerGroup(consumerGroup);
pullRequest.setNextOffset(nextOffset);
pullRequest.setMessageQueue(mq);
pullRequest.setProcessQueue(pq);
pullRequestList.add(pullRequest);
changed = true;
}
} else {
log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq);
}
}
}
this.dispatchPullRequest(pullRequestList);
return changed;
}
负载均衡策略
AllocateMessageQueueStrategy 提供了6种实现
AllocateMachineRoomNearby
机房临近
AllocateMessageQueueAveragely
平均分配。例如队列MQ1、2、3、4、5、6、7、8， 消费者C1、2、3，则C1消费MQ1、2、3，
C2消费MQ4、5、6，C3消费MQ7、8
AllocateMessageQueueAveragelyByCircle
轮询分配。例如队列MQ1、2、3、4、5、6、7、8， 消费者C1、2、3，则C1消费MQ1、4、7，
C2消费MQ2、5、8，C3消费MQ3、6
AllocateMessageQueueByConfig
根据配置消费
AllocateMessageQueueByMachineRoom
机房分配
AllocateMessageQueueConsistentHash
一致性哈希