RocketMQ源码分析之清除消费队列文件

定时清除过期的消息消费队列文件，当消息存储文件被删除后，关于他的topic索引文件即消费队列文件也没有存在的意义了，查找当前broker最小的消息偏移量

// ConsumeQueue removal interval
private int deleteConsumeQueueFilesInterval = 100;

public void run() {
    try {
        this.deleteExpiredFiles();
    } catch (Throwable e) {
        DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e);
    }
}

private void deleteExpiredFiles() {
    int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval();

    long minOffset = DefaultMessageStore.this.commitLog.getMinOffset();
    if (minOffset > this.lastPhysicalMinOffset) {
        this.lastPhysicalMinOffset = minOffset;

        ConcurrentMap> tables = DefaultMessageStore.this.consumeQueueTable;

        for (ConcurrentMap maps : tables.values()) {
            for (ConsumeQueue logic : maps.values()) {
                int deleteCount = logic.deleteExpiredFile(minOffset);

                if (deleteCount > 0 && deleteLogicsFilesInterval > 0) {
                    try {
                        Thread.sleep(deleteLogicsFilesInterval);
                    } catch (InterruptedException ignored) {
                    }
                }
            }
        }

        DefaultMessageStore.this.indexService.deleteExpiredFile(minOffset);
    }
}

根据最小偏移量开始查找过期文件，获取所有的文件，循环查找文件的最后一个索引信息存储的结果，一个文件可以存储30W数据，判断结果中的消息偏移量与当前实际存在的最小的消息偏移量的大小，小于的话就代表该文件已经过期

public int deleteExpiredFile(long offset) {
    int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(offset, CQ_STORE_UNIT_SIZE);
    this.correctMinOffset(offset);
    return cnt;
}

public int deleteExpiredFileByOffset(long offset, int unitSize) {
    Object[] mfs = this.copyMappedFiles(0);

    List files = new ArrayList();
    int deleteCount = 0;
    if (null != mfs) {

        int mfsLength = mfs.length - 1;

        for (int i = 0; i < mfsLength; i++) {
            boolean destroy;
            MappedFile mappedFile = (MappedFile) mfs[i];
            SelectMappedBufferResult result = mappedFile.selectMappedBuffer(this.mappedFileSize - unitSize);
            if (result != null) {
                long maxOffsetInLogicQueue = result.getByteBuffer().getLong();
                result.release();
                destroy = maxOffsetInLogicQueue < offset;
                if (destroy) {
                    log.info("physic min offset " + offset + ", logics in current mappedFile max offset "
                        + maxOffsetInLogicQueue + ", delete it");
                }
            } else if (!mappedFile.isAvailable()) { // Handle hanged file.
                log.warn("Found a hanged consume queue file, attempting to delete it.");
                destroy = true;
            } else {
                log.warn("this being not executed forever.");
                break;
            }

            if (destroy && mappedFile.destroy(1000 * 60)) {
                files.add(mappedFile);
                deleteCount++;
            } else {
                break;
            }
        }
    }

    deleteExpiredFile(files);

    return deleteCount;
}

收集过期的文件进行删除。

void deleteExpiredFile(List files) {

    if (!files.isEmpty()) {

        Iterator iterator = files.iterator();
        while (iterator.hasNext()) {
            MappedFile cur = iterator.next();
            if (!this.mappedFiles.contains(cur)) {
                iterator.remove();
                log.info("This mappedFile {} is not contained by mappedFiles, so skip it.", cur.getFileName());
            }
        }

        try {
            if (!this.mappedFiles.removeAll(files)) {
                log.error("deleteExpiredFile remove failed.");
            }
        } catch (Exception e) {
            log.error("deleteExpiredFile has exception.", e);
        }
    }
}

最后矫正当前消费队列对象中保存最小消息偏移量。

public void correctMinOffset(long phyMinOffset) {
    MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile();
    long minExtAddr = 1;
    if (mappedFile != null) {
        SelectMappedBufferResult result = mappedFile.selectMappedBuffer(0);
        if (result != null) {
            try {
                for (int i = 0; i < result.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
                    long offsetPy = result.getByteBuffer().getLong();
                    result.getByteBuffer().getInt();
                    long tagsCode = result.getByteBuffer().getLong();

                    if (offsetPy >= phyMinOffset) {
                        this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i;
                        log.info("Compute logical min offset: {}, topic: {}, queueId: {}",
                            this.getMinOffsetInQueue(), this.topic, this.queueId);
                        // This maybe not take effect, when not every consume queue has extend file.
                        if (isExtAddr(tagsCode)) {
                            minExtAddr = tagsCode;
                        }
                        break;
                    }
                }
            } catch (Exception e) {
                log.error("Exception thrown when correctMinOffset", e);
            } finally {
                result.release();
            }
        }
    }

    if (isExtReadEnable()) {
        this.consumeQueueExt.truncateByMinAddress(minExtAddr);
    }
}

清除索引文件，查找当前第一个文件的最大下标是否小于要删除的最小偏移量，如果只有一个索引文件的话，说明暂时文件还没有写满，所以在删除文件时忽略最后一个文件，一个文件能存储2000W索引，把符合条件的文件放到集合中

public void deleteExpiredFile(long offset) {
    Object[] files = null;
    try {
        this.readWriteLock.readLock().lock();
        if (this.indexFileList.isEmpty()) {
            return;
        }

        long endPhyOffset = this.indexFileList.get(0).getEndPhyOffset();
        if (endPhyOffset < offset) {
            files = this.indexFileList.toArray();
        }
    } catch (Exception e) {
        log.error("destroy exception", e);
    } finally {
        this.readWriteLock.readLock().unlock();
    }

    if (files != null) {
        List fileList = new ArrayList();
        for (int i = 0; i < (files.length - 1); i++) {
            IndexFile f = (IndexFile) files[i];
            if (f.getEndPhyOffset() < offset) {
                fileList.add(f);
            } else {
                break;
            }
        }

        this.deleteExpiredFile(fileList);
    }
}

删除索引文件

private void deleteExpiredFile(List files) {
    if (!files.isEmpty()) {
        try {
            this.readWriteLock.writeLock().lock();
            for (IndexFile file : files) {
                boolean destroyed = file.destroy(3000);
                destroyed = destroyed && this.indexFileList.remove(file);
                if (!destroyed) {
                    log.error("deleteExpiredFile remove failed.");
                    break;
                }
            }
        } catch (Exception e) {
            log.error("deleteExpiredFile has exception.", e);
        } finally {
            this.readWriteLock.writeLock().unlock();
        }
    }
}