RocketMQ源码（三）：broker的启动（二）

RocketMQ源码（一）：NameServer的启动
RocketMQ源码（二）：broker的启动（一）
RocketMQ源码（四）：producer的启动
RocketMQ源码（五）：producer发送消息
RocketMQ源码（六）：broker接收消息
RocketMQ源码（七）：consumer的启动
RocketMQ源码（八）：consumer消息拉取（一）
RocketMQ源码（九）：consumer消息拉取（二）
上一篇文章说了BrokerController的创建，现在开始分析BrokerController的start过程

public void start() throws Exception {
    // 获取整个文件夹锁对象，保证只有一个 messageStore 操作存储文件
    lock = lockFile.getChannel().tryLock(0, 1, false);
    if (lock == null || lock.isShared() || !lock.isValid()) {
        throw new RuntimeException("Lock failed,MQ already started");
    }

    lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes()));
    lockFile.getChannel().force(true);
    {
        // 获取commitLog中最小的Offset
        long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();
        // 替换为所有consumerQueue中最大的消费offset
        for (ConcurrentMap maps : this.consumeQueueTable.values()) {
            for (ConsumeQueue logic : maps.values()) {
                if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {
                    maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();
                }
            }
        }
        if (maxPhysicalPosInLogicQueue < 0) {
            maxPhysicalPosInLogicQueue = 0;
        }
        // 如果有人删除了所有的 consumequeue 文件或磁盘被损坏。
        if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
            maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
            log.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());
        }
        log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
            maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
        // 设置consumerQueue的构造起始offset
        this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
        // 根据commitLog构造consumerQueue和index的内容
        this.reputMessageService.start();

        // 等待消息分配完毕
        while (true) {
            if (dispatchBehindBytes() <= 0) {
                break;
            }
            Thread.sleep(1000);
            log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());
        }
        // 由于前面的消息分配，这里将ConsumeQueue的Topic和QueueId，以及MaxOffset保存在table中，
        // 同时调用correctMinOffset方法根据物理队列最小offset计算修正逻辑队列最小offset
        this.recoverTopicQueueTable();
    }

    if (!messageStoreConfig.isEnableDLegerCommitLog()) {
        this.haService.start();
        this.handleScheduleMessageService(messageStoreConfig.getBrokerRole());
    }
    // 刷新 consumerQueue 的数据到磁盘上
    this.flushConsumeQueueService.start();
    // commitLog 刷盘服务启动
    this.commitLog.start();
    // 存储统计服务
    this.storeStatsService.start();

    this.createTempFile();
    // 添加定时器
    this.addScheduleTask();
    this.shutdown = false;
}

可以看到brokercontroller的启动，其实也就是启动各种服务的过程，接下来分别看看这些服务都是做什么的。

1. DefaultMessageStore.start

public void start() throws Exception {
    // 获取整个文件夹锁对象，保证只有一个 messageStore 操作存储文件
    lock = lockFile.getChannel().tryLock(0, 1, false);
    if (lock == null || lock.isShared() || !lock.isValid()) {
        throw new RuntimeException("Lock failed,MQ already started");
    }

    lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes()));
    lockFile.getChannel().force(true);
    {
        // 获取commitLog中最小的Offset
        long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();
        // 替换为所有consumerQueue中最大的消费offset
        for (ConcurrentMap maps : this.consumeQueueTable.values()) {
            for (ConsumeQueue logic : maps.values()) {
                if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {
                    maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();
                }
            }
        }
        if (maxPhysicalPosInLogicQueue < 0) {
            maxPhysicalPosInLogicQueue = 0;
        }
        // 如果有人删除了所有的 consumequeue 文件或磁盘被损坏。
        if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
            maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
            log.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());
        }
        log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
            maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
        // 设置consumerQueue的构造起始offset
        this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
        // 根据commitLog构造consumerQueue和index的内容
        this.reputMessageService.start();

        // 等待消息分配完毕
        while (true) {
            if (dispatchBehindBytes() <= 0) {
                break;
            }
            Thread.sleep(1000);
            log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());
        }
        // 由于前面的消息分配，这里将ConsumeQueue的Topic和QueueId，以及MaxOffset保存在table中，
        // 同时调用correctMinOffset方法根据物理队列最小offset计算修正逻辑队列最小offset
        this.recoverTopicQueueTable();
    }

    if (!messageStoreConfig.isEnableDLegerCommitLog()) {
        this.haService.start();
        this.handleScheduleMessageService(messageStoreConfig.getBrokerRole());
    }
    // 刷新 consumerQueue 的数据到磁盘上
    this.flushConsumeQueueService.start();
    // commitLog 刷盘服务启动
    this.commitLog.start();
    // 存储统计服务
    this.storeStatsService.start();

    this.createTempFile();
    // 添加定时器
    this.addScheduleTask();
    this.shutdown = false;
}

这里主要就是整个store文件夹里面文件的构造过程，以及物理文件和逻辑对象之间内容的加载和刷盘等，这里分成3部分来分析

1.1 reputMessageService.start()

ReputMessageService的功能可以概述为：发送消息的再次分发
接下来看看ReputMessageService是如何发挥作用的

public void run() {
    DefaultMessageStore.log.info(this.getServiceName() + " service started");

    while (!this.isStopped()) {
        try {
            Thread.sleep(1);
            this.doReput();
        } catch (Exception e) {
            DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e);
        }
    }

    DefaultMessageStore.log.info(this.getServiceName() + " service end");
}

可以看到这里就是重复的执行doReput方法。doReput方法其实就是从消费的最大offset开始，把未消费的commitLog消息取出来，然后根据取出来的消息不断的构建consumerQueue和indexFile。consumerQueue和indexFile详细的文件结构就不描述了，可通过链接自行了解。

private void doReput() {
if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {
    log.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.",
        this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());
    this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();
}
// isCommitLogAvailable 方法，就是判断 reputFromOffset 是否达到了最后一个文件能访问的地方
for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {
    // 根据偏移量，读取偏移量 到 偏移量+ commitlog文件中有效数据的最大偏移量。如果未找到数据，结束doReput 方法
    if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable()
        && this.reputFromOffset >= DefaultMessageStore.this.getConfirmOffset()) {
        break;
    }
    // 根据reputFromOffset获取commitLog中对应的mappedFile的对应位置起始的所有消息数据
    SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);
    if (result != null) {
        try {
            this.reputFromOffset = result.getStartOffset();
            // 循环从 SelectMappedBufferResult 中读取消息，每次读取一条
            for (int readSize = 0; readSize < result.getSize() && doNext; ) {
                // 从 SelectMappedBufferResult 中读取一条消息，生成 DispatchRequest 对象
                DispatchRequest dispatchRequest =
                    DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false);
                // 这条消息占用的字节数组长度
                int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize();

                if (dispatchRequest.isSuccess()) {
                    if (size > 0) {
                        // 根据 commitlog 文件内容实时构建 consumequeue、index文件的关键所在
                        DefaultMessageStore.this.doDispatch(dispatchRequest);
                        // 如果开启了长轮询并且角色为主节点，则通知有新消息到达，执行一次 pullRequest 验证
                        // 消费端拉取消息的一种方式
                        if (BrokerRole.SLAVE != DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole()
                            && DefaultMessageStore.this.brokerConfig.isLongPollingEnable()) {
                            DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
                                dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
                                dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
                                dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
                        }
                        // 滚动增加reputFromOffset
                        this.reputFromOffset += size;
                        readSize += size;
                        // 从节点进行相应的统计工作
                        if (DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
                            DefaultMessageStore.this.storeStatsService
                                .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).incrementAndGet();
                            DefaultMessageStore.this.storeStatsService
                                .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic())
                                .addAndGet(dispatchRequest.getMsgSize());
                        }
                    }
                    // 如果当前文件没有可读取消息数据，滚动到下一个mappedFile的起始位置
                    else if (size == 0) {
                        this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset);
                        readSize = result.getSize();
                    }
                } else if (!dispatchRequest.isSuccess()) {

                    if (size > 0) {
                        log.error("[BUG]read total count not equals msg total size. reputFromOffset={}", reputFromOffset);
                        this.reputFromOffset += size;
                    } else {
                        doNext = false;
                        // If user open the dledger pattern or the broker is master node,
                        // it will not ignore the exception and fix the reputFromOffset variable
                        if (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog() ||
                            DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) {
                            log.error("[BUG]dispatch message to consume queue error, COMMITLOG OFFSET: {}",
                                this.reputFromOffset);
                            this.reputFromOffset += result.getSize() - readSize;
                        }
                    }
                }
            }
        } finally {
            result.release();
        }
    } else {
        doNext = false;
    }
}
}

这里主要做了三件事：

• 从commitLog读取数据
• 通过CommitLogDispatcher将读取的数据写入到consumerQueue和indexFile中
• 发出消息达到通知，触发在长或短轮询中等待的链接。
  加下来继续看看前两件事，消息达到通知在后续的consumer消息拉取中再继续分析

1.1.1 commitLog.getData

public SelectMappedBufferResult getData(final long offset) {
    return this.getData(offset, offset == 0);
}

public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) {
    // mappedFile大小，默认1G
    int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();
    // 根据 offset 通过mappedFileQueue的findMappedFileByOffset方法定位物理文件的逻辑MappedFile对象
    MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, returnFirstOnNotFound);
    if (mappedFile != null) {
        // 通过offset和mappedFileSize（1G）取余，得到文件指针起始位置
        int pos = (int) (offset % mappedFileSize);
        // 从pos位置开始，读取mappedFile的剩余所有数据
        SelectMappedBufferResult result = mappedFile.selectMappedBuffer(pos);
        return result;
    }
    return null;
}

获取offset对应位置的mappedFile，以及mappedFile从offerset开始到文件结束之间的缓存数据

1.1.2 doDispatch

public void doDispatch(DispatchRequest req) {
    // 这里的dispatcherList，在DefaultMessageStore初始化的时候添加了
    // CommitLogDispatcherBuildConsumeQueue 和 CommitLogDispatcherBuildIndex 实例
    for (CommitLogDispatcher dispatcher : this.dispatcherList) {
        dispatcher.dispatch(req);
    }
}

这里挑CommitLogDispatcherBuildConsumeQueue来分析，看看是如何通过commitLog中的数据构造consumerQueue文件的

class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {

    @Override
    public void dispatch(DispatchRequest request) {
        final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());
        switch (tranType) {
            // 当消息满足 TRANSACTION_NOT_TYPE 和 TRANSACTION_COMMIT_TYPE 时，调用 putMessagePositionInfo 方法
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                DefaultMessageStore.this.putMessagePositionInfo(request);
                break;
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                break;
        }
    }
}

针对普通消息以及提交类的事务消息，需要通过putMessagePositionInfo，将消息的信息刷新到consumerQueue中。其实可以从这里看出来一些事务消息的实现方式，如果是事务消息或者是事务回滚消息，那么是不会在consumerQueue中写入数据的，自然而然consumer也就无法消费对应的数据。

public void putMessagePositionInfo(DispatchRequest dispatchRequest) {
    // 根据 topic 和 queueId 查询到 对应的ConsumeQueue，如果没有则新建
    ConsumeQueue cq = this.findConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());
    cq.putMessagePositionInfoWrapper(dispatchRequest);
}

第一步就是通过topic和queueId查询consumerQueue，如果为空则先创建对应的consumerQueue对象，然后再把消息写入到consumerQueue中

public ConsumeQueue findConsumeQueue(String topic, int queueId) {
    // 通过topic获取queueId-consumerQueue的map对象
    ConcurrentMap map = consumeQueueTable.get(topic);
    if (null == map) {
        ConcurrentMap newMap = new ConcurrentHashMap(128);
        ConcurrentMap oldMap = consumeQueueTable.putIfAbsent(topic, newMap);
        if (oldMap != null) {
            map = oldMap;
        } else {
            map = newMap;
        }
    }

    ConsumeQueue logic = map.get(queueId);
    // 如果没有则新建
    if (null == logic) {
        ConsumeQueue newLogic = new ConsumeQueue(
            topic,
            queueId,
            // consumerQueue 的物理地址
            StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()),
            // 文件大小
            this.getMessageStoreConfig().getMappedFileSizeConsumeQueue(),
            this);
        ConsumeQueue oldLogic = map.putIfAbsent(queueId, newLogic);
        if (oldLogic != null) {
            logic = oldLogic;
        } else {
            logic = newLogic;
        }
    }

    return logic;
}

比较简单继续往下看

public void putMessagePositionInfoWrapper(DispatchRequest request) {
    // 最大的重试次数
    final int maxRetries = 30;
    boolean canWrite = this.defaultMessageStore.getRunningFlags().isCQWriteable();
    for (int i = 0; i < maxRetries && canWrite; i++) {
        long tagsCode = request.getTagsCode();
        // 是否写入consumerQueueExt数据
        if (isExtWriteEnable()) {
            ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
            cqExtUnit.setFilterBitMap(request.getBitMap());
            cqExtUnit.setMsgStoreTime(request.getStoreTimestamp());
            cqExtUnit.setTagsCode(request.getTagsCode());

            long extAddr = this.consumeQueueExt.put(cqExtUnit);
            if (isExtAddr(extAddr)) {
                tagsCode = extAddr;
            } else {
                log.warn("Save consume queue extend fail, So just save tagsCode! {}, topic:{}, queueId:{}, offset:{}", cqExtUnit,
                    topic, queueId, request.getCommitLogOffset());
            }
        }
        // 将 commitLog 的offset、消息大小、tag信息写入consumerQueue文件
        boolean result = this.putMessagePositionInfo(request.getCommitLogOffset(),
            request.getMsgSize(), tagsCode, request.getConsumeQueueOffset());
        // 记录消息写入时间
        if (result) {
            if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE ||
                this.defaultMessageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {
                this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp());
            }
            this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp());
            return;
        } else {
            // XXX: warn and notify me
            log.warn("[BUG]put commit log position info to " + topic + ":" + queueId + " " + request.getCommitLogOffset()
                + " failed, retry " + i + " times");

            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                log.warn("", e);
            }
        }
    }

    // XXX: warn and notify me
    log.error("[BUG]consume queue can not write, {} {}", this.topic, this.queueId);
    this.defaultMessageStore.getRunningFlags().makeLogicsQueueError();
}

这里就是在for循环中重复最多30次，直到将 commitLog 的offset、消息大小、tag信息通过putMessagePositionInfo写入consumerQueue文件中

private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
    final long cqOffset) {
    if (offset + size <= this.maxPhysicOffset) {
        log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset);
        return true;
    }
    // ConsumeQueue文件是通过20字节来存放对应CommitLog文件中的消息映射
    this.byteBufferIndex.flip();
    this.byteBufferIndex.limit(CQ_STORE_UNIT_SIZE);
    this.byteBufferIndex.putLong(offset);
    this.byteBufferIndex.putInt(size);
    this.byteBufferIndex.putLong(tagsCode);
    // 根据 ConsumeQueueOffset 即 cqOffset * CQ_STORE_UNIT_SIZE（20）计算expectLogicOffset
    final long expectLogicOffset = cqOffset * CQ_STORE_UNIT_SIZE;
    // expectLogicOffset就是ConsumeQueue文件逻辑Offset，由此可以通过getLastMappedFile找到对应的文件映射MappedFile
    MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(expectLogicOffset);
    if (mappedFile != null) {
        // 第一个mappedFile，并且是写入的第一个数据
        if (mappedFile.isFirstCreateInQueue() && cqOffset != 0 && mappedFile.getWrotePosition() == 0) {
            // 设置最小的逻辑地址
            this.minLogicOffset = expectLogicOffset;
            // 设置刷盘起始位置
            this.mappedFileQueue.setFlushedWhere(expectLogicOffset);
            // 设置数据提交位置
            this.mappedFileQueue.setCommittedWhere(expectLogicOffset);
            // 填充一个空数据在文件最开始处
            this.fillPreBlank(mappedFile, expectLogicOffset);
            log.info("fill pre blank space " + mappedFile.getFileName() + " " + expectLogicOffset + " "
                + mappedFile.getWrotePosition());
        }
        if (cqOffset != 0) {
            long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset();

            if (expectLogicOffset < currentLogicOffset) {
                log.warn("Build  consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}",
                    expectLogicOffset, currentLogicOffset, this.topic, this.queueId, expectLogicOffset - currentLogicOffset);
                return true;
            }

            if (expectLogicOffset != currentLogicOffset) {
                LOG_ERROR.warn(
                    "[BUG]logic queue order maybe wrong, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}",
                    expectLogicOffset,
                    currentLogicOffset,
                    this.topic,
                    this.queueId,
                    expectLogicOffset - currentLogicOffset
                );
            }
        }
        // 最大的物理地址
        this.maxPhysicOffset = offset + size;
        // 将 byteBufferIndex 追加到 mappedFile 后
        return mappedFile.appendMessage(this.byteBufferIndex.array());
    }
    return false;
}

至此消息就已经写入到了consumerQueue中，接下来回到DefaultMessageStore的start方法中

1.2 handleScheduleMessageService

if (!messageStoreConfig.isEnableDLegerCommitLog()) {
    this.haService.start();
    this.handleScheduleMessageService(messageStoreConfig.getBrokerRole());
}

这里如果没有开启 DLegerCommitLog 的存储消息模式，就会执行下面的两个方法
haService与rocketmq的高可用架构实现有关暂不分析

public void handleScheduleMessageService(final BrokerRole brokerRole) {
    if (this.scheduleMessageService != null) {
        if (brokerRole == BrokerRole.SLAVE) {
            this.scheduleMessageService.shutdown();
        } else {
            this.scheduleMessageService.start();
        }
    }
}

可以看到这里判断如果broker的角色是Mater那么就会启动scheduleMessageService服务，scheduleMessageService是实现延迟消息的核心服务，简单来说就是consumer的延迟消息都会进入一个 SCHEDULE_TOPIC_XXXX 开头的topic中（消息推送过程中会提到相关部分），然后scheduleMessageService通过一个定时器判断消息是否到了延迟时间，如果到了就会把消息从 SCHEDULE_TOPIC_XXXX 中取出，重新投递到consumer的消费队列中。
先简单的认识下ScheduleMessageService，看下它的属性

// 第一次执行时的延迟时间
private static final long FIRST_DELAY_TIME = 1000L;
// 处理完一次consumerQueue所有消息时暂停时间
private static final long DELAY_FOR_A_WHILE = 100L;
// 发送异常时的消息处理时间
private static final long DELAY_FOR_A_PERIOD = 10000L;
// 延迟等级和对应延迟时间
private final ConcurrentMap delayLevelTable = new ConcurrentHashMap(32);
// 延迟等级和对应consumerQueue下次处理的offset
private final ConcurrentMap offsetTable = new ConcurrentHashMap(32);
private final DefaultMessageStore defaultMessageStore;
private final AtomicBoolean started = new AtomicBoolean(false);
private Timer timer;
private MessageStore writeMessageStore;
private int maxDelayLevel;

加下来看看它是如何实现延迟消息的，看下它的start方法

public void start() {
    // 标识已经start
    if (started.compareAndSet(false, true)) {
        this.timer = new Timer("ScheduleMessageTimerThread", true);
        for (Map.Entry entry : this.delayLevelTable.entrySet()) {
            Integer level = entry.getKey();
            Long timeDelay = entry.getValue();
            Long offset = this.offsetTable.get(level);
            if (null == offset) {
                offset = 0L;
            }
            // 每个延迟等级使用一个DeliverDelayedMessageTimerTask，在1s之后执行
            if (timeDelay != null) {
                this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME);
            }
        }

        this.timer.scheduleAtFixedRate(new TimerTask() {
            @Override
            public void run() {
                try {
                    if (started.get()) ScheduleMessageService.this.persist();
                } catch (Throwable e) {
                    log.error("scheduleAtFixedRate flush exception", e);
                }
            }
        }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval());
    }
}

这里有两部分的内容

• 处理延迟队列中的消息
• 持久化offsetTable（每个延迟consumerQueue的）到磁盘C:\Users\xxx\store\config\delayOffset.json文件中
  继续看下DeliverDelayedMessageTimerTask这个线程做了什么事情

@Override
public void run() {
    try {
        if (isStarted()) {
            this.executeOnTimeup();
        }
    } catch (Exception e) {
        // XXX: warn and notify me
        log.error("ScheduleMessageService, executeOnTimeup exception", e);
        ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(
            this.delayLevel, this.offset), DELAY_FOR_A_PERIOD);
    }
}

public void executeOnTimeup() {
    // 获取延迟等级对应的consumerQueue
    ConsumeQueue cq = ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(
            TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel2QueueId(delayLevel));

    long failScheduleOffset = offset;

    if (cq != null) {
        SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(this.offset);
        if (bufferCQ != null) {
            try {
                long nextOffset = offset;
                int i = 0;
                ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
                // 循环的取出每一条延迟消息
                for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
                    long offsetPy = bufferCQ.getByteBuffer().getLong();
                    int sizePy = bufferCQ.getByteBuffer().getInt();
                    // 注意这个tagCode，不再是普通的tag的hashCode，而是该延时消息到期的时间
                    long tagsCode = bufferCQ.getByteBuffer().getLong();

                    if (cq.isExtAddr(tagsCode)) {
                        if (cq.getExt(tagsCode, cqExtUnit)) {
                            tagsCode = cqExtUnit.getTagsCode();
                        } else {
                            //can't find ext content.So re compute tags code.
                            log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}",
                                tagsCode, offsetPy, sizePy);
                            long msgStoreTime = defaultMessageStore.getCommitLog().pickupStoreTimestamp(offsetPy, sizePy);
                            tagsCode = computeDeliverTimestamp(delayLevel, msgStoreTime);
                        }
                    }
                    long now = System.currentTimeMillis();
                    long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode);
                    // 下次需要判断的offset
                    nextOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
                    // 延迟时间减去当前时间
                    long countdown = deliverTimestamp - now;
                    // 到延迟时间
                    if (countdown <= 0) {
                        // 从commitLog取出延迟消息
                        MessageExt msgExt = ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset(offsetPy, sizePy);
                        // 如果数据取出
                        if (msgExt != null) {
                            try {
                                // MessageExt对象转换成MessageExtBrokerInner对象
                                MessageExtBrokerInner msgInner = this.messageTimeup(msgExt);
                                // 判断消息的真实队列是否是半事务队列
                                if (TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC.equals(msgInner.getTopic())) {
                                    log.error("[BUG] the real topic of schedule msg is {}, discard the msg. msg={}", msgInner.getTopic(), msgInner);
                                    continue;
                                }
                                // 把延迟消息存入真实消息队列
                                PutMessageResult putMessageResult = ScheduleMessageService.this.writeMessageStore.putMessage(msgInner);

                                if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {
                                    continue;
                                } else {
                                    // 如果消息放入失败，结束当前task，重新投递DeliverDelayedMessageTimerTask从当前offset再次开始
                                    log.error("ScheduleMessageService, a message time up, but reput it failed, topic: {} msgId {}",
                                            msgExt.getTopic(), msgExt.getMsgId());
                                    ScheduleMessageService.this.timer.schedule(
                                            new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), DELAY_FOR_A_PERIOD);
                                    ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
                                    return;
                                }
                            } catch (Exception e) {
                                log.error("ScheduleMessageService, messageTimeup execute error, drop it. msgExt="
                                        + msgExt + ", nextOffset=" + nextOffset + ",offsetPy="
                                        + offsetPy + ",sizePy=" + sizePy, e);
                            }
                        }
                    }
                    // 未到延迟时间
                    else {
                        // 设定到达延迟时间之后执行当前消息
                        ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), countdown);
                        // 更新下一个需要处理的延迟消息
                        ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
                        return;
                    }
                } // end of for
                nextOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
                // 处理完当前MappedFile中的消息后，重新启动TimerTask，从下一个消息开始处理
                ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(
                    this.delayLevel, nextOffset), DELAY_FOR_A_WHILE);
                ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
                return;
            } finally {

                bufferCQ.release();
            }
        } // end of if (bufferCQ != null)
        else {
            long cqMinOffset = cq.getMinOffsetInQueue();
            if (offset < cqMinOffset) {
                failScheduleOffset = cqMinOffset;
                log.error("schedule CQ offset invalid. offset=" + offset + ", cqMinOffset="
                    + cqMinOffset + ", queueId=" + cq.getQueueId());
            }
        }
    } // end of if (cq != null)
    ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(this.delayLevel,
        failScheduleOffset), DELAY_FOR_A_WHILE);
}

总的来说就是根据offset从对应的consumerQueue中取出所有的数据，然后遍历这些数据如果达到了延迟时间则把消息投递到真实队列中，如果没有到延迟时间则结束当前任务，重新投递DeliverDelayedMessageTimerTask在延时时间到达时从offset再次开始执行。
接下来回到DefaultMessageStore的start方法中

1.3 flushConsumeQueueService.start

主要是用于将内存中的consumerQueue刷新到磁盘上

public void run() {
    DefaultMessageStore.log.info(this.getServiceName() + " service started");

    while (!this.isStopped()) {
        try {
            // 默认1秒
            int interval = DefaultMessageStore.this.getMessageStoreConfig().getFlushIntervalConsumeQueue();
            this.waitForRunning(interval);
            // 默认只尝试一次
            this.doFlush(1);
        } catch (Exception e) {
            DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e);
        }
    }

    this.doFlush(RETRY_TIMES_OVER);

    DefaultMessageStore.log.info(this.getServiceName() + " service end");
}

private void doFlush(int retryTimes) {
    int flushConsumeQueueLeastPages = DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueLeastPages();
    // 服务停止时，把consumerQueue的所有数据刷到磁盘
    if (retryTimes == RETRY_TIMES_OVER) {
        flushConsumeQueueLeastPages = 0;
    }

    long logicsMsgTimestamp = 0;
    // 刷盘时间
    int flushConsumeQueueThoroughInterval = DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueThoroughInterval();
    long currentTimeMillis = System.currentTimeMillis();
    // 如果到了刷盘时间，不管consumerQueue数据量是否到达刷盘限制，都进行刷盘
    if (currentTimeMillis >= (this.lastFlushTimestamp + flushConsumeQueueThoroughInterval)) {
        this.lastFlushTimestamp = currentTimeMillis;
        flushConsumeQueueLeastPages = 0;
        logicsMsgTimestamp = DefaultMessageStore.this.getStoreCheckpoint().getLogicsMsgTimestamp();
    }

    ConcurrentMap> tables = DefaultMessageStore.this.consumeQueueTable;

    for (ConcurrentMap maps : tables.values()) {
        for (ConsumeQueue cq : maps.values()) {
            boolean result = false;
            for (int i = 0; i < retryTimes && !result; i++) {
                // 刷盘
                result = cq.flush(flushConsumeQueueLeastPages);
            }
        }
    }
    // 保存最后的刷盘时间
    if (0 == flushConsumeQueueLeastPages) {
        if (logicsMsgTimestamp > 0) {
            DefaultMessageStore.this.getStoreCheckpoint().setLogicsMsgTimestamp(logicsMsgTimestamp);
        }
        DefaultMessageStore.this.getStoreCheckpoint().flush();
    }
}

当满足两种情况时调用ConsumerQueue.flush方法，进行刷盘

• 达到flushConsumeQueueThoroughInterval刷盘时间
• consumerQueue中的数据量达到此次刷盘所需的page量
  继续看下ConsumerQueue的flush方法

public boolean flush(final int flushLeastPages) {
    boolean result = this.mappedFileQueue.flush(flushLeastPages);
    if (isExtReadEnable()) {
        result = result & this.consumeQueueExt.flush(flushLeastPages);
    }
    return result;
}

其实也就是调用mappedFileQueue的flush方法

public boolean flush(final int flushLeastPages) {
    boolean result = true;
    // 根据上次刷新的位置，得到当前的 MappedFile 对象
    MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0);
    if (mappedFile != null) {
        long tmpTimeStamp = mappedFile.getStoreTimestamp();
        // 执行 MappedFile 的 flush 方法
        int offset = mappedFile.flush(flushLeastPages);
        long where = mappedFile.getFileFromOffset() + offset;
        result = where == this.flushedWhere;
        // 更新上次刷新的位置
        this.flushedWhere = where;
        if (0 == flushLeastPages) {
            this.storeTimestamp = tmpTimeStamp;
        }
    }

    return result;
}
回到DefaultMessageStore的start方法中

1.4 commitLog.start

先来看下commitLog的属性

// Message's MAGIC CODE daa320a7
public final static int MESSAGE_MAGIC_CODE = -626843481;
protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
// End of file empty MAGIC CODE cbd43194
protected final static int BLANK_MAGIC_CODE = -875286124;
// 真实消息的逻辑队列
protected final MappedFileQueue mappedFileQueue;
// 消息存储服务
protected final DefaultMessageStore defaultMessageStore;
// 刷commitLog数据到磁盘文件的服务
private final FlushCommitLogService flushCommitLogService;
private final FlushCommitLogService commitLogService;

private final AppendMessageCallback appendMessageCallback;
private final ThreadLocal batchEncoderThreadLocal;
protected HashMap topicQueueTable = new HashMap(1024);
protected volatile long confirmOffset = -1L;

private volatile long beginTimeInLock = 0;

protected final PutMessageLock putMessageLock;

可以把commitLog理解为一个包装，用于处理消息的存取，消息的持久化策略等，接下来看看它的持久化方法

public void start() {
    // 默认异步刷盘
    this.flushCommitLogService.start();
    // 是否开启暂存池，异步刷盘
    if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
        this.commitLogService.start();
    }
}

这里的FlushCommitLogService有三个实现类，对应了commitLog的三种刷盘方式：

• CommitRealTimeService：异步使用暂存池刷盘
• FlushRealTimeService：异步不使用暂存池刷盘
• GroupCommitService：同步刷盘
  和consumerQueue的刷盘方法大同小异，就不详细的展开说了
  继续回到DefaultMessageStore

1.5 addScheduleTask

private void addScheduleTask() {
    // 定期清除文件，会定期删除掉长时间（默认72小时）未被引用的CommitLog文件
    // 并且删除对应的 consumerQueue 和 indexFile
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            DefaultMessageStore.this.cleanFilesPeriodically();
        }
    }, 1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS);
    // 定期检查CommitLog和ConsumeQueue文件有否损坏、丢失，做日志打印
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            DefaultMessageStore.this.checkSelf();
        }
    }, 1, 10, TimeUnit.MINUTES);
    // 定期虚拟机堆栈使用日志记录
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            if (DefaultMessageStore.this.getMessageStoreConfig().isDebugLockEnable()) {
                try {
                    if (DefaultMessageStore.this.commitLog.getBeginTimeInLock() != 0) {
                        long lockTime = System.currentTimeMillis() - DefaultMessageStore.this.commitLog.getBeginTimeInLock();
                        if (lockTime > 1000 && lockTime < 10000000) {

                            String stack = UtilAll.jstack();
                            final String fileName = System.getProperty("user.home") + File.separator + "debug/lock/stack-"
                                + DefaultMessageStore.this.commitLog.getBeginTimeInLock() + "-" + lockTime;
                            MixAll.string2FileNotSafe(stack, fileName);
                        }
                    }
                } catch (Exception e) {
                }
            }
        }
    }, 1, 1, TimeUnit.SECONDS);

    // this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    // @Override
    // public void run() {
    // DefaultMessageStore.this.cleanExpiredConsumerQueue();
    // }
    // }, 1, 1, TimeUnit.HOURS);
    this.diskCheckScheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        public void run() {
            DefaultMessageStore.this.cleanCommitLogService.isSpaceFull();
        }
    }, 1000L, 10000L, TimeUnit.MILLISECONDS);
}

之后回到brokerController，接下来是remotingServer和fastRemotingServer的启动，这里在NameServer的启动过程中已经分析过了，不在赘述。值得一提的是虽然启动是一样的不过broker的processorTable绑定了对应事件的处理器，因此不会使用DefaultRequestProcessor来处理provider和consumer客户端的事件

1.6 brokerOuterAPI.start

BrokerOuterAPI用于处理Broker到NameSrv的各种连接事件

public void start() {
    // 创建 EventExecutorGroup
    this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(
        nettyClientConfig.getClientWorkerThreads(),
        new ThreadFactory() {

            private AtomicInteger threadIndex = new AtomicInteger(0);

            @Override
            public Thread newThread(Runnable r) {
                return new Thread(r, "NettyClientWorkerThread_" + this.threadIndex.incrementAndGet());
            }
        });

    Bootstrap handler = this.bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class)
        .option(ChannelOption.TCP_NODELAY, true)
        .option(ChannelOption.SO_KEEPALIVE, false)
        .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, nettyClientConfig.getConnectTimeoutMillis())
        .option(ChannelOption.SO_SNDBUF, nettyClientConfig.getClientSocketSndBufSize())
        .option(ChannelOption.SO_RCVBUF, nettyClientConfig.getClientSocketRcvBufSize())
        .handler(new ChannelInitializer() {
            @Override
            public void initChannel(SocketChannel ch) throws Exception {
                ChannelPipeline pipeline = ch.pipeline();
                if (nettyClientConfig.isUseTLS()) {
                    if (null != sslContext) {
                        pipeline.addFirst(defaultEventExecutorGroup, "sslHandler", sslContext.newHandler(ch.alloc()));
                        log.info("Prepend SSL handler");
                    } else {
                        log.warn("Connections are insecure as SSLContext is null!");
                    }
                }
                pipeline.addLast(
                    defaultEventExecutorGroup,
                    new NettyEncoder(),
                    new NettyDecoder(),
                    // 指定空闲时间后断开连接
                    new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()),
                    // 当nettyclient监听到Netty事件，比如连接、断开、异常等情况，执行客户端自定义的事件处理
                    new NettyConnectManageHandler(),
                    new NettyClientHandler());
            }
        });
    // 过滤超时的response
    this.timer.scheduleAtFixedRate(new TimerTask() {
        @Override
        public void run() {
            try {
                NettyRemotingClient.this.scanResponseTable();
            } catch (Throwable e) {
                log.error("scanResponseTable exception", e);
            }
        }
    }, 1000 * 3, 1000);

    if (this.channelEventListener != null) {
        this.nettyEventExecutor.start();
    }
}

启动netty客户端，处理和NameSrv通信的事件。
回到brokerController，pullRequestHoldService是处理消息拉取相关请求的（后续分析）。

1.7 clientHousekeepingService.start

public void start() {

    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            try {
                ClientHousekeepingService.this.scanExceptionChannel();
            } catch (Throwable e) {
                log.error("Error occurred when scan not active client channels.", e);
            }
        }
    }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
}

private void scanExceptionChannel() {
    this.brokerController.getProducerManager().scanNotActiveChannel();
    this.brokerController.getConsumerManager().scanNotActiveChannel();
    this.brokerController.getFilterServerManager().scanNotActiveChannel();
}

清除异常的Channel缓存
回到BrokerController，接下来是如下代码

// 启动 mqfiltersrv.exe
if (this.filterServerManager != null) {
    this.filterServerManager.start();
}
// 向 NameSrv 注册自己
if (!messageStoreConfig.isEnableDLegerCommitLog()) {
    // Master会启动事务消息检查，遍历未提交、未回滚的部分消息并向生产者发送检查请求以获取事务状态
    // 进行偏移量的检查和计算等操作，并移除掉需要丢弃的消息
    startProcessorByHa(messageStoreConfig.getBrokerRole());
    // Slave会启动同步操作
    handleSlaveSynchronize(messageStoreConfig.getBrokerRole());
    this.registerBrokerAll(true, false, true);
}

这里重点看看registerBrokerAll方法

1.8 registerBrokerAll

public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) {
    // 封装 topicConfigTable 到 TopicConfigSerializeWrapper 中
    TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();

    if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
        || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
        ConcurrentHashMap topicConfigTable = new ConcurrentHashMap();
        for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) {
            TopicConfig tmp =
                new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
                    this.brokerConfig.getBrokerPermission());
            topicConfigTable.put(topicConfig.getTopicName(), tmp);
        }
        topicConfigWrapper.setTopicConfigTable(topicConfigTable);
    }
    // 判断是否需要注册，和NameSrv比较Broker的version是否变化
    if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
        this.getBrokerAddr(),
        this.brokerConfig.getBrokerName(),
        this.brokerConfig.getBrokerId(),
        this.brokerConfig.getRegisterBrokerTimeoutMills())) {
        doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);
    }
}

首先是包装 ConcurrentMap topicConfigTable = new ConcurrentHashMap(1024);
然后通过 needRegister 判断是否需要需要想Namesrv注册

if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
    this.getBrokerAddr(),
    this.brokerConfig.getBrokerName(),
    this.brokerConfig.getBrokerId(),
    this.brokerConfig.getRegisterBrokerTimeoutMills())) {
    doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper);
}

private boolean needRegister(final String clusterName,
    final String brokerAddr,
    final String brokerName,
    final long brokerId,
    final int timeoutMills) {
    // 封装topicManage到topicConfigWrapper中
    TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();
    List changeList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigWrapper, timeoutMills);
    boolean needRegister = false;
    // 只要相比任何一个NameServer有了改变，就需要注册
    for (Boolean changed : changeList) {
        if (changed) {
            needRegister = true;
            break;
        }
    }
    return needRegister;
}

public List needRegister(
    final String clusterName,
    final String brokerAddr,
    final String brokerName,
    final long brokerId,
    final TopicConfigSerializeWrapper topicConfigWrapper,
    final int timeoutMills) {
    final List changedList = new CopyOnWriteArrayList<>();
    // 获取NameServer的地址列表
    List nameServerAddressList = this.remotingClient.getNameServerAddressList();
    if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
        // CountDownLatch 用于控制和每个NameServer比较完成
        final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
        // 遍历 NameServer
        // 循环比较每个NameServer上和当前broker的dataVersion，并且记录每个比较的结果
        for (final String namesrvAddr : nameServerAddressList) {
            brokerOuterExecutor.execute(new Runnable() {
                @Override
                public void run() {
                    try {
                        // 封装QueryDataVersionRequestHeader请求头
                        QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
                        requestHeader.setBrokerAddr(brokerAddr);
                        requestHeader.setBrokerId(brokerId);
                        requestHeader.setBrokerName(brokerName);
                        requestHeader.setClusterName(clusterName);
                        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
                        request.setBody(topicConfigWrapper.getDataVersion().encode());
                        // 向NameServer发送 QUERY_DATA_VERSION 类型的请求，在NameServer端比较dataVersion
                        RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills);
                        DataVersion nameServerDataVersion = null;
                        Boolean changed = false;
                        switch (response.getCode()) {
                            case ResponseCode.SUCCESS: {
                                QueryDataVersionResponseHeader queryDataVersionResponseHeader =
                                    (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
                                changed = queryDataVersionResponseHeader.getChanged();
                                byte[] body = response.getBody();
                                if (body != null) {
                                    nameServerDataVersion = DataVersion.decode(body, DataVersion.class);
                                    if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) {
                                        changed = true;
                                    }
                                }
                                if (changed == null || changed) {
                                    changedList.add(Boolean.TRUE);
                                }
                            }
                            default:
                                break;
                        }
                        log.warn("Query data version from name server {} OK,changed {}, broker {},name server {}", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? "" : nameServerDataVersion);
                    } catch (Exception e) {
                        changedList.add(Boolean.TRUE);
                        log.error("Query data version from name server {}  Exception, {}", namesrvAddr, e);
                    } finally {
                        countDownLatch.countDown();
                    }
                }
            });

        }
        try {
            countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            log.error("query dataversion from nameserver countDownLatch await Exception", e);
        }
    }
    return changedList;
}

当判断为需要注册之后，就会调用doRegisterBrokerAll方法，将broker信息注册到NameServer上

private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway,
    TopicConfigSerializeWrapper topicConfigWrapper) {
    // 返回的是向每个NameServer注册的结果，其中包含了haServerAddr、masterAddr、kvTable等信息
    List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll(
        this.brokerConfig.getBrokerClusterName(),
        this.getBrokerAddr(),
        this.brokerConfig.getBrokerName(),
        this.brokerConfig.getBrokerId(),
        this.getHAServerAddr(),
        topicConfigWrapper,
        this.filterServerManager.buildNewFilterServerList(),
        oneway,
        this.brokerConfig.getRegisterBrokerTimeoutMills(),
        this.brokerConfig.isCompressedRegister());

    if (registerBrokerResultList.size() > 0) {
        RegisterBrokerResult registerBrokerResult = registerBrokerResultList.get(0);
        if (registerBrokerResult != null) {
            // 更新主从节点信息
            if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) {
                this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr());
            }
            // 保存master地址信息
            this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr());
            // 更新配置信息
            if (checkOrderConfig) {
                this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable());
            }
        }
    }
}

接下来brokerController会启动一个定时器定时调用registerBrokerAll以便实时更新信息
接下来还会启动两个定时器

// broker的统计，待实现
if (this.brokerStatsManager != null) {
    this.brokerStatsManager.start();
}
// 当读写磁盘忙碌的时候，结束掉一些发送消息的请求
if (this.brokerFastFailure != null) {
    this.brokerFastFailure.start();
}

继续看下brokerFastFailure

public void start() {
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            if (brokerController.getBrokerConfig().isBrokerFastFailureEnable()) {
                cleanExpiredRequest();
            }
        }
    }, 1000, 10, TimeUnit.MILLISECONDS);
}

private void cleanExpiredRequest() {
    // 如果os内存缓存繁忙
    // 快速失败，返回发送消息请求队列中的请求
    while (this.brokerController.getMessageStore().isOSPageCacheBusy()) {
        try {
            if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) {
                final Runnable runnable = this.brokerController.getSendThreadPoolQueue().poll(0, TimeUnit.SECONDS);
                if (null == runnable) {
                    break;
                }

                final RequestTask rt = castRunnable(runnable);
                rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", System.currentTimeMillis() - rt.getCreateTimestamp(), this.brokerController.getSendThreadPoolQueue().size()));
            } else {
                break;
            }
        } catch (Throwable ignored) {
        }
    }
    // 结束对应队列中，超过对应等待时间的请求
    cleanExpiredRequestInQueue(this.brokerController.getSendThreadPoolQueue(),
        this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue());
    cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(),
        this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue());
    cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(),
        this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue());
    cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this
        .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue());
}

比较简单就不详细分析了
到此broker的启动过程就结束了