Rocketmq consumer源码

1、启动comsumer

非常复杂,挑几个重要的来掰掰

org.apache.rocketmq.client.consumer.DefaultMQPushConsumer#start

org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl#start
    
public synchronized void start() throws MQClientException {
        switch (this.serviceState) {
            case CREATE_JUST:
                log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(),
                    this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode());
                this.serviceState = ServiceState.START_FAILED;
                //校验配置参数
                this.checkConfig();
                //广播和集群模式下Topic数据通过拷贝数据进行同步的
                this.copySubscription();
                //consumer实例命名
                if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) {
                    this.defaultMQPushConsumer.changeInstanceNameToPID();
                }
                //comsumer单机部署多个实例,一个实例一个工厂管理
                //多机器部署,一个机器就是一个实例,一个工厂管理
                this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook);

//设置负载均衡策略                this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup());
                this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel());
                this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy());
                this.rebalanceImpl.setmQClientFactory(this.mQClientFactory);
                //初始化consumer消费包装器,包装消费整个netty client 发起请求过程。
                if (this.pullAPIWrapper == null) {
                    this.pullAPIWrapper = new PullAPIWrapper(
                        mQClientFactory,
                        this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode());
                }
                //通过钩子hook实现消息增强,非常简单,没有任何代理技术,就是个钩子回调。
                this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList);
                //广播模式下,本地处理offset
                //集群模式下,broker处理offset
                if (this.defaultMQPushConsumer.getOffsetStore() != null) {
                    this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();
                } else {
                    switch (this.defaultMQPushConsumer.getMessageModel()) {
                        case BROADCASTING:
                            this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                            break;
                        case CLUSTERING:
                            this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
                            break;
                        default:
                            break;
                    }
                    this.defaultMQPushConsumer.setOffsetStore(this.offsetStore);
                }
                this.offsetStore.load();
                //注册comsumer顺序监听MessageListenerOrderly、并发监听MessageListenerConcurrently
                if (this.getMessageListenerInner() instanceof MessageListenerOrderly) {
                    this.consumeOrderly = true;
                    this.consumeMessageService =
                        new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());
                    //POPTODO reuse Executor ?
                    this.consumeMessagePopService = new ConsumeMessagePopOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner());
                } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) {
                    this.consumeOrderly = false;
                    this.consumeMessageService =
                        new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());
                    //POPTODO reuse Executor ?
                    this.consumeMessagePopService =
                        new ConsumeMessagePopConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner());
                }

                this.consumeMessageService.start();
                // POPTODO
                this.consumeMessagePopService.start();
                boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this);
                if (!registerOK) {
                    this.serviceState = ServiceState.CREATE_JUST;
                    this.consumeMessageService.shutdown(defaultMQPushConsumer.getAwaitTerminationMillisWhenShutdown());
                    throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup()
                        + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL),
                        null);
                }
                //启动MQClientInstance,初始化重平衡
                mQClientFactory.start();
                log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup());
                this.serviceState = ServiceState.RUNNING;
                break;
            case RUNNING:
            case START_FAILED:
            case SHUTDOWN_ALREADY:
                throw new MQClientException("The PushConsumer service state not OK, maybe started once, "
                    + this.serviceState
                    + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK),
                    null);
            default:
                break;
        }
        //订阅更新topic变化操作
        this.updateTopicSubscribeInfoWhenSubscriptionChanged();
        this.mQClientFactory.checkClientInBroker();
        //发送心跳包到broker
        this.mQClientFactory.sendHeartbeatToAllBrokerWithLock();
        //comsumer负载均衡
        this.mQClientFactory.rebalanceImmediately();
    }

2、consumer消费消息

org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl#pullMessage=》

org.apache.rocketmq.client.impl.consumer.PullAPIWrapper#pullKernelImpl=》

org.apache.rocketmq.client.impl.MQClientAPIImpl#pullMessage=》

异步

 org.apache.rocketmq.client.impl.MQClientAPIImpl#pullMessageAsync

org.apache.rocketmq.remoting.netty.NettyRemotingClient#invokeAsync

同步

org.apache.rocketmq.client.impl.MQClientAPIImpl#pullMessageSync

org.apache.rocketmq.remoting.netty.NettyRemotingClient#invokeSync

单次不会触发

实现,底层实际就是netty client 发送请求交互,详细可以看mq producer。

3、offset处理

4、重平衡(负载均衡)

发生时机:

mQClientFactory.start();

this.mQClientFactory.rebalanceImmediately();

org.apache.rocketmq.client.impl.consumer.RebalanceImpl#doRebalance
balanced = this.rebalanceByTopic(topic, isOrder);
    private boolean rebalanceByTopic(final String topic, final boolean isOrder) {
        boolean balanced = true;
        switch (messageModel) {
            //广播模式下,consumer消费完namesrv的消息,不需要负载均衡,只是更新下那台机器进行负载就可以了。
            case BROADCASTING: {
                Set mqSet = this.topicSubscribeInfoTable.get(topic);
                if (mqSet != null) {
                    boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder);
                    if (changed) {
                        this.messageQueueChanged(topic, mqSet, mqSet);
                        log.info("messageQueueChanged {} {} {} {}", consumerGroup, topic, mqSet, mqSet);
                    }

                    balanced = mqSet.equals(getWorkingMessageQueue(topic));
                } else {
                    this.messageQueueChanged(topic, Collections.emptySet(), Collections.emptySet());
                    log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
                }
                break;
            }
            case CLUSTERING: {
                //获取messagequeue
                Set mqSet = this.topicSubscribeInfoTable.get(topic);
                //根据topic和消费组适配到指定cid,
                List cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup);
                if (null == mqSet) {
                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                        this.messageQueueChanged(topic, Collections.emptySet(), Collections.emptySet());
                        log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
                    }
                }

                if (null == cidAll) {
                    log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic);
                }

                if (mqSet != null && cidAll != null) {
                    List mqAll = new ArrayList();
                    mqAll.addAll(mqSet);
                    //排序:用于确保messagequeue只会被一台consumer消费,cid和messagequeue绑定。
                    Collections.sort(mqAll);
                    Collections.sort(cidAll);
                    //负载均衡策略
                    //默认平均分配
                    //环形平均策略
                    //一致性hash策略
                    //同机房策略
                    //就近机房策略
                    AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy;

                    List allocateResult = null;
                    try {
                        allocateResult = strategy.allocate(
                            this.consumerGroup,
                            this.mQClientFactory.getClientId(),
                            mqAll,
                            cidAll);
                    } catch (Throwable e) {
                        log.error("allocate message queue exception. strategy name: {}, ex: {}", strategy.getName(), e);
                        return false;
                    }

                    Set allocateResultSet = new HashSet();
                    if (allocateResult != null) {
                        allocateResultSet.addAll(allocateResult);
                    }
                    //发生变更的消息队列,需要更新负载均衡信息.
                    boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder);
                    if (changed) {
                        log.info(
                            "client rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, rebalanceResultSize={}, rebalanceResultSet={}",
                            strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(),
                            allocateResultSet.size(), allocateResultSet);
                        this.messageQueueChanged(topic, mqSet, allocateResultSet);
                    }

                    balanced = allocateResultSet.equals(getWorkingMessageQueue(topic));
                }
                break;
            }
            default:
                break;
        }

        return balanced;
    }

你可能感兴趣的:(RocketMq,rocketmq)