Rabbitmq源码分析，重复消费问题的redis或数据库代码实现
目录
底层源码解析
自定义唯一id算法
MessageProperties类的相关实现
自定义消息ID生成器
配置和使用
Rabbitmq是怎么判断是不是重复消息的呢？
通过Redis的幂等性处理
消息消费者实现
分布式锁实现的重复检测
完整的消息处理流程
基于数据库实现
Mapper接口
消息处理服务
RabbitMQ消息消费者
底层源码解析
RabbitMQ判断重复消息主要通过消息的唯一标识（MessageId）和幂等性处理来实现
我们该如何给消息添加唯一ID呢？
其实很简单，SpringAMQP的MessageConverter自带了MessageID的功能，我们只要开启这个功能即可。
以Jackson的消息转换器为例：
@Bean
public MessageConverter messageConverter(){
// 1.定义消息转换器
Jackson2JsonMessageConverter jjmc = new Jackson2JsonMessageConverter();
// 2.配置自动创建消息id，用于识别不同消息，也可以在业务中基于ID判断是否是重复消息
jjmc.setCreateMessageIds(true);
return jjmc;
}不配置会有默认的消息转换器SimpleMessageConverter，我们可以进入到源码中看一下
可以看到默认是new了SimpleMessageConverter作为转换器，当然也可以像之前代码那般配置转换器。
我把SimpleMessageConverter 代码简化成以下代码，其实toMessage方法是SimpleMessageConverter父类的父类的方法。
public class SimpleMessageConverter implements MessageConverter {
@Override
public Message toMessage(Object object, MessageProperties messageProperties)
throws MessageConversionException {
// 如果没有messageProperties，创建一个新的
if (messageProperties == null) {
messageProperties = new MessageProperties();
}
// 生成消息ID（如果没有的话）
if (messageProperties.getMessageId() == null) {
messageProperties.setMessageId(UUID.randomUUID().toString());
}
// ... 其他转换逻辑
}
}完整代码如下：
SimpleMessageConverter的父类为AllowedListDeserializingMessageConverter，进入到AllowedListDeserializingMessageConverter类中。
再进入到AllowedListDeserializingMessageConverter类的父类AbstractMessageConverter中
可以看到 AbstractMessageConverter中的tomessage方法，就此SimpleMessageConverter也继承了该方法，可以看到方法里面生成的MessageId是用uuid生成的。
不过SimpleMessageConverter 对于要发送的消息体 body 为 byte[] 时不进行处理，如果是 String 则转成字节数组,如果是 Java 对象，则使用 jdk 序列化将消息转成字节数组，转出来的结果较大，含class类名，类相应方法等信息，因此性能较差。
Rabbitmq也提供了 Jackson2JsonMessageConverter 来支持消息内容 JSON 序列化与反序列化，所以我们一般会用Jackson2JsonMessageConverter。
Jackson2JsonMessageConverter的源码很简单，我们进入他的父类AbstractJackson2MessageConverter看看
可以看到最后还是会继承AbstractMessageConverter，所以本质上生成的MessageId也是用uuid生成的。
自定义唯一id算法
MessageProperties类的相关实现
public class MessageProperties {
private String messageId;
private String correlationId;
private String replyTo;
private Long timestamp;
// 设置消息ID
public void setMessageId(String messageId) {
this.messageId = messageId;
}
// 获取消息ID
public String getMessageId() {
return this.messageId;
}
// 生成默认消息ID
public void generateMessageId() {
if (this.messageId == null) {
this.messageId = UUID.randomUUID().toString();
}
}
}自定义消息ID生成器
public class CustomMessageIdGenerator implements MessageIdGenerator {
private final AtomicLong sequence = new AtomicLong(0);
private final String prefix;
public CustomMessageIdGenerator(String prefix) {
this.prefix = prefix;
}
@Override
public String generateId() {
return prefix + "-" + System.currentTimeMillis() + "-" +
sequence.incrementAndGet();
}
}
// 使用示例
public class CustomMessageConverter implements MessageConverter {
private final MessageIdGenerator messageIdGenerator;
public CustomMessageConverter(MessageIdGenerator messageIdGenerator) {
this.messageIdGenerator = messageIdGenerator;
}
@Override
public Message toMessage(Object object, MessageProperties messageProperties) {
if (messageProperties == null) {
messageProperties = new MessageProperties();
}
if (messageProperties.getMessageId() == null) {
messageProperties.setMessageId(messageIdGenerator.generateId());
}
// ... 其他转换逻辑
}
}雪花算法实现唯一id
public class DistributedMessageIdGenerator implements MessageIdGenerator {
private final SnowflakeIdWorker snowflake;
public DistributedMessageIdGenerator(long workerId, long datacenterId) {
this.snowflake = new SnowflakeIdWorker(workerId, datacenterId);
}
@Override
public String generateId() {
return String.valueOf(snowflake.nextId());
}
}
// 雪花算法实现
public class SnowflakeIdWorker {
private final long workerId;
private final long datacenterId;
private long sequence = 0L;
private long lastTimestamp = -1L;
// 各部分占位数
private final long workerIdBits = 5L;
private final long datacenterIdBits = 5L;
private final long sequenceBits = 12L;
public SnowflakeIdWorker(long workerId, long datacenterId) {
this.workerId = workerId;
this.datacenterId = datacenterId;
}
public synchronized long nextId() {
long timestamp = timeGen();
// 时钟回拨检测
if (timestamp < lastTimestamp) {
throw new RuntimeException("Clock moved backwards");
}
// 同一毫秒内序列号自增
if (lastTimestamp == timestamp) {
sequence = (sequence + 1) & ((1 << sequenceBits) - 1);
if (sequence == 0) {
timestamp = tilNextMillis(lastTimestamp);
}
} else {
sequence = 0L;
}
lastTimestamp = timestamp;
// 组合ID
return ((timestamp - twepoch) << timestampLeftShift)
| (datacenterId << datacenterIdShift)
| (workerId << workerIdShift)
| sequence;
}
private long tilNextMillis(long lastTimestamp) {
long timestamp = timeGen();
while (timestamp <= lastTimestamp) {
timestamp = timeGen();
}
return timestamp;
}
private long timeGen() {
return System.currentTimeMillis();
}
}配置和使用
@Configuration
public class RabbitConfig {
@Bean
public MessageConverter messageConverter() {
SimpleMessageConverter converter = new SimpleMessageConverter();
// 可以配置自定义的ID生成器
// converter.setMessageIdGenerator(new CustomMessageIdGenerator("APP"));
return converter;
}
@Bean
public RabbitTemplate rabbitTemplate(ConnectionFactory connectionFactory) {
RabbitTemplate template = new RabbitTemplate(connectionFactory);
template.setMessageConverter(messageConverter());
return template;
}
}
// 使用示例
@Service
public class MessageService {
@Autowired
private RabbitTemplate rabbitTemplate;
public void sendMessage(String message) {
// 消息ID会自动生成
rabbitTemplate.convertAndSend("exchange", "routingKey", message);
}
}Rabbitmq是怎么判断是不是重复消息的呢？
通过Redis的幂等性处理
@Component
public class MessageDeduplicationHandler {
@Autowired
private RedisTemplate redisTemplate;
private static final String DEDUP_KEY_PREFIX = "mq:dedup:";
private static final long DEDUP_EXPIRATION = 24 * 60 * 60; // 24小时
/**
* 检查消息是否重复
*/
public boolean isDuplicate(String messageId) {
String dedupKey = DEDUP_KEY_PREFIX + messageId;
// 使用Redis的setNX命令，如果key不存在则设置成功返回true
Boolean isFirstConsume = redisTemplate.opsForValue()
.setIfAbsent(dedupKey, "1", DEDUP_EXPIRATION, TimeUnit.SECONDS);
return !Boolean.TRUE.equals(isFirstConsume);
}
} 消息消费者实现
@Component
public class MessageConsumer {
@Autowired
private MessageDeduplicationHandler deduplicationHandler;
@RabbitListener(queues = "myQueue")
public void handleMessage(Message message,
@Header(AmqpHeaders.MESSAGE_ID) String messageId,
Channel channel) throws IOException {
try {
// 检查消息是否重复
if (deduplicationHandler.isDuplicate(messageId)) {
log.info("重复消息，messageId: {}", messageId);
// 确认消息已处理
channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
return;
}
// 处理消息
processMessage(message);
// 确认消息
channel.basicAck(message.getMessageProperties().getDeliveryTag(), false);
} catch (Exception e) {
// 处理异常，可能需要重试或死信队列
channel.basicNack(message.getMessageProperties().getDeliveryTag(),
false, true);
}
}
}分布式锁实现的重复检测
@Component
public class DistributedMessageDeduplication {
@Autowired
private RedissonClient redissonClient;
private static final String LOCK_PREFIX = "mq:lock:";
public boolean processWithLock(String messageId, Runnable processor) {
RLock lock = redissonClient.getLock(LOCK_PREFIX + messageId);
try {
// 尝试获取锁
if (lock.tryLock(5, 10, TimeUnit.SECONDS)) {
try {
// 检查是否已处理
if (isMessageProcessed(messageId)) {
return false;
}
// 处理消息
processor.run();
// 记录处理状态
markMessageAsProcessed(messageId);
return true;
} finally {
lock.unlock();
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return false;
}
}完整的消息处理流程
@Component
@Slf4j
public class RobustMessageConsumer {
@Autowired
private DistributedMessageDeduplication deduplication;
@Autowired
private MessageRecordService messageRecordService;
@RabbitListener(queues = "myQueue")
public void onMessage(Message message, Channel channel) throws IOException {
String messageId = message.getMessageProperties().getMessageId();
long deliveryTag = message.getMessageProperties().getDeliveryTag();
try {
boolean processed = deduplication.processWithLock(messageId, () -> {
// 业务处理逻辑
processBusinessLogic(message);
// 记录消息处理状态
messageRecordService.recordMessage(messageId);
});
if (processed) {
log.info("消息处理成功: {}", messageId);
channel.basicAck(deliveryTag, false);
} else {
log.info("消息已经处理过: {}", messageId);
channel.basicAck(deliveryTag, false);
}
} catch (Exception e) {
log.error("消息处理失败: {}", messageId, e);
// 根据异常类型决定是否重试
boolean requeue = shouldRequeue(e);
channel.basicNack(deliveryTag, false, requeue);
}
}
private boolean shouldRequeue(Exception e) {
// 根据异常类型判断是否需要重试
return !(e instanceof BusinessException);
}
}基于数据库实现
@Data
@TableName("message_record")
public class MessageRecord {
@TableId(type = IdType.INPUT)
private String messageId;
private String businessKey;
private String status; // NEW, PROCESSING, COMPLETED, FAILED
private Integer retryCount;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
}Mapper接口
@Mapper
public interface MessageRecordMapper extends BaseMapper {
@Select("SELECT * FROM message_record WHERE message_id = #{messageId} FOR UPDATE")
MessageRecord selectByIdForUpdate(@Param("messageId") String messageId);
@Update("UPDATE message_record SET status = #{status}, " +
"retry_count = retry_count + 1, " +
"update_time = NOW() " +
"WHERE message_id = #{messageId}")
int updateStatus(@Param("messageId") String messageId, @Param("status") String status);
@Delete("DELETE FROM message_record WHERE create_time < #{threshold}")
int deleteOldRecords(@Param("threshold") LocalDateTime threshold);
} 消息处理服务
@Service
@Slf4j
public class MessageProcessService {
@Autowired
private MessageRecordMapper messageRecordMapper;
@Transactional(rollbackFor = Exception.class)
public boolean processMessage(String messageId, String businessKey,
MessageProcessor processor) {
try {
// 1. 检查消息是否存在且已处理
MessageRecord record = messageRecordMapper.selectByIdForUpdate(messageId);
if (record != null) {
if ("COMPLETED".equals(record.getStatus())) {
log.info("消息已处理完成，跳过处理: {}", messageId);
return true;
}
if (record.getRetryCount() >= 3) {
log.warn("消息重试次数超限: {}", messageId);
messageRecordMapper.updateStatus(messageId, "FAILED");
return false;
}
} else {
// 2. 创建新的消息记录
record = new MessageRecord();
record.setMessageId(messageId);
record.setBusinessKey(businessKey);
record.setStatus("NEW");
record.setRetryCount(0);
messageRecordMapper.insert(record);
}
// 3. 更新状态为处理中
messageRecordMapper.updateStatus(messageId, "PROCESSING");
// 4. 执行业务处理
processor.process();
// 5. 更新状态为完成
messageRecordMapper.updateStatus(messageId, "COMPLETED");
return true;
} catch (Exception e) {
log.error("消息处理失败: {}", messageId, e);
messageRecordMapper.updateStatus(messageId, "FAILED");
throw e;
}
}
}
@FunctionalInterface
interface MessageProcessor {
void process() throws Exception;
}RabbitMQ消息消费者
@Component
@Slf4j
public class RabbitMessageConsumer {
@Autowired
private MessageProcessService messageProcessService;
@RabbitListener(queues = "myQueue")
public void onMessage(Message message, Channel channel) throws IOException {
String messageId = message.getMessageProperties().getMessageId();
String businessKey = message.getMessageProperties().getHeaders()
.get("businessKey").toString();
long deliveryTag = message.getMessageProperties().getDeliveryTag();
try {
boolean processed = messageProcessService.processMessage(
messageId,
businessKey,
() -> {
// 实际的业务处理逻辑
String messageBody = new String(message.getBody());
processBusinessLogic(messageBody);
}
);
if (processed) {
channel.basicAck(deliveryTag, false);
} else {
// 处理失败，发送到死信队列
channel.basicNack(deliveryTag, false, false);
}
} catch (Exception e) {
log.error("消息处理异常: {}", messageId, e);
// 根据异常类型决定是否重试
boolean requeue = shouldRequeue(e);
channel.basicNack(deliveryTag, false, requeue);
}
}
private void processBusinessLogic(String messageBody) {
// 业务处理逻辑
}
private boolean shouldRequeue(Exception e) {
return !(e instanceof BusinessException);
}
}