五、Rabbitmq消息可靠性投递实践
思路：将投递失败的消息存入数据库，利用定时任务去定时重试发送消息，重试三次，如果三次都无法投递的话，这时候需要人工干预，去处理消息为什么没有能够投递成功
1. 基础配置
SpringBoot版本：2.0.3
- sql
create table if not exists rabbit.msg_broker
(
id int auto_increment primary key,
retry tinyint default '0' not null comment '尝试投递次数',
status tinyint default '1' not null comment '此时的消息的状态',
reason varchar(1000) default '' not null comment '失败原因',
create_time timestamp default CURRENT_TIMESTAMP not null,
update_time timestamp default CURRENT_TIMESTAMP not null,
corr_id varchar(64) default '' not null comment '消息唯一标识',
message varchar(10000) default '' not null comment '消息体',
rounting_key varchar(20) default '' not null comment '路由键',
constraint msg_broker_corr_id_uindex unique (corr_id)
)comment 'rabbit消息信息';
实体类
package com.itcloud.entity;
import java.util.Date;
public class MsgBroker {
//主键
private Integer id;
//消息唯一标识
private String corrId;
//消息尝试发送次数
private Integer retry;
//消息状态
private Integer status;
//消息发送失败原因
private String reason;
//创建时间
private Date createTime;
//更新时间
private Date updateTime;
//消息体
private String message;
//消息路由键
private String rountingKey;
}
- RabbitApplication.java
这里添加注解：
@EnableScheduling开启springBoot定时任务功能
/**
* rabbbit 消息可靠性投递最佳实践
* @author ITCloud
*/
@SpringBootApplication
@EnableScheduling
public class RabbitApplication {
public static void main(String[] args) {
SpringApplication.run(RabbitApplication.class, args);
}
}
- application.yml
server:
port: 8077
spring:
rabbitmq:
addresses: 192.168.186.130
port: 5672
username: itcloud
password: itcloud
virtual-host: hmall
listener:
direct:
acknowledge-mode: manual # 设置手动签收
datasource:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/rabbit?useSSL=false
username: root
password: HelloJava0903!
- RabbitConfig
常量配置类
package com.itcloud.config;
public interface RabbitConfig {
//其中一个交换机名称
public static final String EXCHANGE_DIRECT = "hmex.direct";
public static final int SENDING_STATUS = 0; //消息正在重试发送
public static final int SUCCESS_STATUS = 1; // 消息发送成功
public static final int FAIL_STATUS = 2; // 消息发送失败
public static final int NOT_FOUNT = -1; //数据不存在
public static final int FIRST_RETRY = 1; //第一次重试
public static final int MAX_RETRY = 3; //重试上线
}
2. 数据库层
数据库层使用JdbcTempalte操作数据库，如果项目中可以直接换成Mybatis等ORM框架
package com.itcloud.dao;
import java.util.List;
import java.util.Map;
import com.itcloud.entity.MsgBroker;
public interface IMsgBrokerDao {
int selectStatus(String correlationDataId);
int updateRetry(String correlationDataId);
int insertMsgBroker(MsgBroker msgBroker);
int updateStatus(String correlationDataId, int status);
int updateIfExit(String correlationDataId);
/**
* 查询出status < 3的错误消息
* @return
*/
List<Map<String, Object>> selectByStatus();
}
package com.itcloud.dao.impl;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import com.itcloud.config.RabbitConfig;
import com.itcloud.dao.IMsgBrokerDao;
import com.itcloud.entity.MsgBroker;
/**
* 消息投递失败之后，进行消息重试机制
*
* @author ITCloud
*
*/
@Repository
public class MsgBrokerDaoImpl implements IMsgBrokerDao {
@Autowired
private JdbcTemplate jdbcTemplate;
public int selectStatus(String correlationDataId) {
String sql = "SELECT retry from msg_broker where corr_id = ?";
try {
Integer retry = jdbcTemplate.queryForObject(sql, new Object[] { correlationDataId }, Integer.class);
return retry;
} catch (Exception e) {
return RabbitConfig.NOT_FOUNT;
}
}
public int updateRetry(String correlationDataId) {
String sql = "update msg_broker set retry = retry + 1 where corr_id = ?";
return jdbcTemplate.update(sql, new Object[] { correlationDataId });
}
public int updateStatus(String correlationDataId, int status) {
String sql = "update msg_broker set status = ? where corr_id = ?";
return jdbcTemplate.update(sql, new Object[] { status, correlationDataId });
}
public int insertMsgBroker(MsgBroker msgBroker) {
String sql = "insert msg_broker(corr_Id, status, reason, message, rounting_key) values(?,?,?,?,?)";
try {
return jdbcTemplate.update(sql, new Object[] { msgBroker.getCorrId(), msgBroker.getStatus(),
msgBroker.getReason(), msgBroker.getMessage(), msgBroker.getRountingKey() });
} catch (Exception e) { //唯一约束异常,目的就是让其插入失败，
return 0;
}
}
@Override
public int updateIfExit(String correlationDataId) {
String sql = "update msg_broker set status = 2 where corr_id = ?";
try {
return jdbcTemplate.update(sql, new Object[] { correlationDataId, correlationDataId });
} catch (Exception e) {
return RabbitConfig.NOT_FOUNT;
}
}
@Override
public List<Map<String, Object>> selectByStatus() {
String sql = "select * from msg_broker where status != 3";
List<Map<String, Object>> data = jdbcTemplate.queryForList(sql);
return data;
}
}
3. 消息投递自定义配置
public interface IRabbitSend {
/**
* direct类型的交换机发送消息, 用于异常消息重新发送
* @param routingKey 路由键
* @param message 消息体
* @param correlationDataId 消息唯一标识
* @return
*/
String sendDirectMsg(String routingKey, Object message, String correlationDataId);
/**
* direct类型交换机消息发送
* @param routingKey
* @param message
* @return
*/
String sendDirectMsg(String routingKey, Object message);
}
两个实现类
package com.itcloud.service.impl;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.support.CorrelationData;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.itcloud.config.RabbitConfig;
import com.itcloud.service.IRabbitSend;
/**
* 普通消息投递，不进行消息确认
* @author ITCloud
*
*/
@Service(value = "rabbitSend")
public class RabbitSend implements IRabbitSend {
@Autowired
private RabbitTemplate rabbitTemplate;
/**
* 尽管这里使用Object来传输，但是开发过程中基本都是json
*/
@Override
public String sendDirectMsg(String routingKey, Object message, String correlationDataId) {
rabbitTemplate.convertAndSend(RabbitConfig.EXCHANGE_DIRECT, routingKey, message);
return null;
}
@Override
public String sendDirectMsg(String routingKey, Object message) {
return this.sendDirectMsg(routingKey, message, null);
}
}
package com.itcloud.service.impl;
import org.slf4j.LoggerFactory;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.rabbit.support.CorrelationData;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.itcloud.config.RabbitConfig;
import com.itcloud.dao.IMsgBrokerDao;
import com.itcloud.entity.MsgBroker;
import com.itcloud.service.IRabbitSend;
/**
需要进行消息确认
*/
@Service(value = "rabbitConfirmSend")
public class RabbitConfirmSend implements IRabbitSend {
private static Logger log = LoggerFactory.getLogger(RabbitConfirmSend.class);
/**
* 用于缓存correlationDataId
*/
private ConcurrentMap<String, Object> curMap = new ConcurrentHashMap<>();
@Autowired
private IMsgBrokerDao msgBrokerDao;
@Autowired
private RabbitTemplate rabbitTemplate;
@PostConstruct
public void init() {
// 消息确认机制
((CachingConnectionFactory) this.rabbitTemplate.getConnectionFactory()).setPublisherConfirms(true);
// 这是一个同步的消息确认回调，同一时刻只有一个消息才能进行消息确认
rabbitTemplate.setConfirmCallback((correlationData, ack, cause) -> {
if (ack) { // 消息投递成功
log.info("Message send success, correLationData:{}", correlationData.getId());
//对于失败的消息，投递成功之后，更改状态
msgBrokerDao.updateIfExit(correlationData.getId());
this.curMap.remove(correlationData.getId());
} else { // 消息投递失败, 将消息存入数据库重新消费
Object[] msgInfo = (Object[]) this.curMap.get(correlationData.getId());
log.error("Message send failure. CorrelationID:{}, cause:{}, message is:{}", correlationData.getId(),
cause, msgInfo[2]);
saveMssage(correlationData.getId(), cause, (String) msgInfo[1], msgInfo[2]);
}
});
}
@Override
public String sendDirectMsg(String routingKey, Object message) {
return this.sendDirectMsg(routingKey, message, UUID.randomUUID().toString());
}
@Override
public String sendDirectMsg(String routingKey, Object message, String correlationDataId) {
CorrelationData correlationData = new CorrelationData(correlationDataId);
// 缓存ID,当消息投递失败的时候，从缓存中获取出来,进行重发
curMap.put(correlationDataId, new Object[] { "D", routingKey, message });
rabbitTemplate.convertAndSend(RabbitConfig.EXCHANGE_DIRECT, routingKey, message, correlationData);
log.info("Message send {}, message is :{}", correlationData.getId(), message);
return correlationDataId;
}
/**
* 将未能消费的消息存入数据库，使用定时任务进行重新消费
*/
private void saveMssage(String correlationDataId, String reason, String rountingKey, Object message) {
MsgBroker msgBroker = new MsgBroker();
msgBroker.setCorrId(correlationDataId);
msgBroker.setStatus(RabbitConfig.SENDING_STATUS);
// 这里讲消息体也存在数据库,如果数量有的很大，有的很小，这对数据库很是浪费，这里可以将消息静态化。
// 当消息发送成功之后，可以清楚静态化文件
msgBroker.setMessage((String) message);
msgBroker.setRountingKey(rountingKey);
msgBroker.setReason(reason);
msgBrokerDao.insertMsgBroker(msgBroker);
this.curMap.remove(correlationDataId);
}
}
4.定时任务
定时任务主要是对数据库中未投递成功的消息进行重新投递
package com.itcloud.config;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.itcloud.dao.IMsgBrokerDao;
import com.itcloud.service.IRabbitSend;
/**
* 定时获取数据库失败的消息队列
* @author ITCloud
*/
@Component
public class RetryMessageTasker {
private Logger log = LoggerFactory.getLogger(RetryMessageTasker.class);
@Resource(name = "rabbitConfirmSend")
private IRabbitSend rabbitSend;
@Autowired
private IMsgBrokerDao msgBrokerDao;
/**
* 表示项目启动5（initialDelay）秒后开始执行，每隔10（fixedDelay）秒执行一次
*/
@Scheduled(initialDelay = 5000, fixedDelay = 10000)
public void retrySendFailMessage() {
log.info("任务开始执行...");
List<Map<String, Object>> lists = msgBrokerDao.selectByStatus();
if (lists != null && lists.size() > 0) {
lists.forEach(msgBrokerMap -> {
String id = (String)msgBrokerMap.get("corr_id");
if ((int)msgBrokerMap.get("retry") >= RabbitConfig.MAX_RETRY) {
msgBrokerDao.updateStatus(id, RabbitConfig.FAIL_STATUS);
} else {
//更新重试次数
msgBrokerDao.updateRetry(id);
//消息重发
rabbitSend.sendDirectMsg((String)msgBrokerMap.get("rounting_key"), msgBrokerMap.get("message"), id);
}
});
}
}
}
总结：
在生产环境中，交换机和队列都会提前建立好，所以在这里，发送消息的时候，我们可以建立sendDirectMsg()、sendTopicMsg()…类似这样的发送方法定义消息发送模板。上述的代码，还可以进行再优化，例如把消息存入数据库很不合适的，假如我们发送一条消息包含几万条数据，数据库的对应的字段应该设置多大合适呢？很难确定，所以对消息做静态化，将消息保存在json文件中，在消息发送的时候可以重新发序列化。或者，也可以将消息保存在其他交换机的队列中，例如我们可以建立一个requeueExchange来回收没有投递成功的消息。比较推荐第二种方式。