Springboot整合RocketMQ
- springBoot 2.2.4
- rocketMq 4.3.2 (服务端)
- rocketMq 4.4.0 (客户端)
- lombok
配置
org.springframework.boot
spring-boot-starter-web
org.apache.rocketmq
rocketmq-client
4.4.0
org.projectlombok
lombok
topic :主题名称
tag :标签，用于过滤
key :消息唯一标示，可以是业务字段组合
body :消息体,字节数组
简单消息示例
import org.apache.rocketmq.common.message.Message;
public class JmsConfig {
//指定NameServer地址，多个地址以 ; 隔开
//public static final String NAME_SERVER = "192.168.100.141:9876;192.168.100.142:9876;192.168.100.149:9876";
public static final String NAME_SERVER = "192.168.0.13:9876";
public static final String DEFAULT_TOPIC = "default_topic_110";
public static final String DEFAULT_ORDERLY_TOPIC = "default_orderly_topic_110";
public static final String DEFAULT_TRANSACTION_TOPIC = "default_transaction_topic_110";
public static final String DEFAULT_TAGS = "default_tags";
//producerGroup
public static String PRODUCER_GROUP = "simple_producer_group";
public static String ORDERLY_PRODUCER_GROUP = "orderly_producer_group";
public static String TRANSACTION_PRODUCER_GROUP = "transaction_producer_group";
//consumerGroup
public static String CONSUMER_GROUP = "simple_consumer_group";
public static String CONSUMER_ORDERLY_GROUP = "orderly_consumer_group";
public static String CONSUMER_TRANSACTION_GROUP = "transaction_consumer_group";
public static class Utils {
public static Message buildDefaultMessage (String msg) {
Message message = new Message(JmsConfig.DEFAULT_TOPIC,DEFAULT_TAGS, msg.getBytes());
return message;
}
public static Message buildOrderlyMessage(String msg) {
Message message = new Message(JmsConfig.DEFAULT_ORDERLY_TOPIC,DEFAULT_TAGS, msg.getBytes());
return message;
}
public static Message buildTransactionMessage(String msg) {
Message message = new Message(JmsConfig.DEFAULT_TRANSACTION_TOPIC,DEFAULT_TAGS, msg.getBytes());
return message;
}
}
}
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
/**
* @author zhang3
* 消息 生产者
*/
@Slf4j
public class MqProducer {
protected String getProducerGroup() {
return JmsConfig.PRODUCER_GROUP;
}
private DefaultMQProducer producer;
public DefaultMQProducer getProducer(){
return this.producer;
}
/**
* 对象在使用之前必须要调用一次，只能初始化一次
*/
@PostConstruct
public void start(){
producer = new DefaultMQProducer(getProducerGroup());
producer.setNamesrvAddr(JmsConfig.NAME_SERVER);
try {
this.producer.start();
} catch (MQClientException e) {
e.printStackTrace();
}
}
/**
* 一般在应用上下文，使用上下文监听器，进行关闭
*/
@PreDestroy
public void shutdown(){
this.producer.shutdown();
}
/**
* 同步 发送消息
*
* @param message 消息
* @return 发送结果
*/
public SendResult sendMsg(Message message) {
try {
SendResult sendResult = getProducer().send(message);
log.info("MQ消息发送成功!");
return sendResult;
} catch (Exception e) {
throw new RuntimeException("MQ消息发送失败!", e);
}
}
/**
* 异步 发送消息
*
* @param message 消息
*/
public void sendAsyMsg(Message message) {
try {
getProducer().send(message, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
log.info("MQ消息发送成功!");
}
@Override
public void onException(Throwable throwable) {
//TODO 补偿操作，根据具体业务定制
throw new RuntimeException("MQ消息发送失败!", throwable);
}
});
} catch (Exception e) {
throw new RuntimeException("MQ消息发送失败!", e);
}
}
/**
* 延迟 发送消息
*
* @param message 消息
* @param delayTimeLevel 延迟级别
* @return 发送结果
*/
public SendResult sendDelayMsg(Message message, int delayTimeLevel) {
try {
//xxx是级别，1表示配置里面的第一个级别，2表示第二个级别，延迟消费级别
//"1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h"
message.setDelayTimeLevel(delayTimeLevel);
SendResult sendResult = getProducer().send(message);
log.info("MQ消息发送成功!");
return sendResult;
} catch (Exception e) {
throw new RuntimeException("MQ消息发送失败!", e);
}
}
/**
* 延迟 发送消息
*
* @param message 消息
* @return 发送结果
*/
public SendResult sendDelayMsg(Message message) {
return sendDelayMsg(message, 3);
}
}
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import javax.annotation.PostConstruct;
import java.io.UnsupportedEncodingException;
/**
* 并发进行消费 消费者
*/
@Slf4j
public class MqConsumer {
private DefaultMQPushConsumer consumer;
@PostConstruct
public void ini() throws MQClientException {
consumer = new DefaultMQPushConsumer(JmsConfig.CONSUMER_GROUP);
consumer.setNamesrvAddr(JmsConfig.NAME_SERVER);
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
consumer.subscribe(JmsConfig.DEFAULT_TOPIC, "*");
// consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
// try {
// Message msg = msgs.get(0);
// System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), new String(msgs.get(0).getBody()));
// String topic = msg.getTopic();
// String body = new String(msg.getBody(), "utf-8");
// String tags = msg.getTags();
// String keys = msg.getKeys();
// System.out.println("topic=" + topic + ", tags=" + tags + ", keys=" + keys + ", msg=" + body);
// return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
// } catch (UnsupportedEncodingException e) {
// e.printStackTrace();
// return ConsumeConcurrentlyStatus.RECONSUME_LATER;
// }
// });
consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
MessageExt msg = msgs.get(0);
int times = msg.getReconsumeTimes();
if (times > 0) {
log.info("重试次数="+times);
}
try {
String topic = msg.getTopic();
String body = new String(msg.getBody(), "utf-8");
String tags = msg.getTags();
String keys = msg.getKeys();
log.info("消费消息:" + "topic=" + topic + ", tags=" + tags + ", keys=" + keys + ", msg=" + body);
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
} catch (UnsupportedEncodingException e) {
log.error("消费异常" + e.getMessage());
//如果重试2次不成功，则记录，人工介入
if(times >= 2){
log.error("重试次数大于2，记录数据库，发短信通知开发人员或者运营人员");
//TODO 记录数据库，发短信通知开发人员或者运营人员
//告诉broker，消息成功
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
}
return ConsumeConcurrentlyStatus.RECONSUME_LATER;
}
});
consumer.start();
log.info("consumer start ...");
}
}
import org._33web.cloud.rocket.mq.jms.orderly.MqOrderlyConsumer;
import org._33web.cloud.rocket.mq.jms.orderly.MqOrderlyProducer;
import org._33web.cloud.rocket.mq.jms.simple.MqConsumer;
import org._33web.cloud.rocket.mq.jms.simple.MqProducer;
import org._33web.cloud.rocket.mq.jms.transaction.DemoTransactionListener;
import org._33web.cloud.rocket.mq.jms.transaction.MqTransactionConsumer;
import org._33web.cloud.rocket.mq.jms.transaction.MqTransactionProducer;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author zhang3
* description TODO
*/
@Configuration
public class JmsBeanConfig {
@Bean
@ConditionalOnMissingBean(name = "mqProducer")
public MqProducer mqProducer() {
return new MqProducer();
}
@Bean
@ConditionalOnMissingBean(name = "mqConsumer")
public MqConsumer mqConsumer() {
return new MqConsumer();
}
@Bean
@ConditionalOnMissingBean(name = "mqOrderlyProducer")
public MqOrderlyProducer mqOrderlyProducer() {
return new MqOrderlyProducer();
}
@Bean
@ConditionalOnMissingBean(name = "mqOrderlyConsumer")
public MqOrderlyConsumer mqOrderlyConsumer() {
return new MqOrderlyConsumer();
}
@Bean
@ConditionalOnMissingBean(name = "mqTransactionProducer")
public MqTransactionProducer mqTransactionProducer() {
return new MqTransactionProducer(new DemoTransactionListener());
}
@Bean
@ConditionalOnMissingBean(name = "mqTransactionConsumer")
public MqTransactionConsumer mqTransactionConsumer() {
return new MqTransactionConsumer();
}
}
有序消息示例
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org._33web.cloud.rocket.mq.jms.simple.MqProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
/**
* @author zhang3
* description 有序消息 生产者
*/
@Slf4j
public class MqOrderlyProducer extends MqProducer {
@Override
protected String getProducerGroup() {
return JmsConfig.ORDERLY_PRODUCER_GROUP;
}
/**
* 选择队列 发送消息
*
* @param message 消息
* @param id 编号
* @return 发送结果
*/
public SendResult sendQueueSelectorMsg(Message message, Integer id) {
try {
SendResult sendResult = getProducer().send(message, (queueLists, msg, o) -> {
Integer num = (Integer) o;
//消息队列(0-3)，默认topic下queue数量是4
Integer queueNum = num % queueLists.size();
return queueLists.get(queueNum);
}, id);
log.info("MQ消息发送成功!");
return sendResult;
} catch (Exception e) {
throw new RuntimeException("MQ消息发送失败!", e);
}
}
}
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import javax.annotation.PostConstruct;
import java.io.UnsupportedEncodingException;
/**
* @author zhang3
* description 有序的进行消费 消费者
*
* 需要配合有序的消息进行一起使用
* {@link MqOrderlyProducer#sendQueueSelectorMsg(Message, Integer)}
*/
@Slf4j
public class MqOrderlyConsumer {
private DefaultMQPushConsumer consumer;
@PostConstruct
public void ini() throws MQClientException {
consumer = new DefaultMQPushConsumer(JmsConfig.CONSUMER_ORDERLY_GROUP);
consumer.setNamesrvAddr(JmsConfig.NAME_SERVER);
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
//默认是集群方式，可以更改为广播，但是广播方式不支持重试
consumer.setMessageModel(MessageModel.CLUSTERING);
consumer.subscribe(JmsConfig.DEFAULT_ORDERLY_TOPIC, "*");
consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {
MessageExt msg = msgs.get(0);
int times = msg.getReconsumeTimes();
if (times > 0) {
log.info("重试次数="+times);
}
try {
// System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), new String(msgs.get(0).getBody()));
String topic = msg.getTopic();
String body = new String(msg.getBody(), "utf-8");
String tags = msg.getTags();
String keys = msg.getKeys();
int queueId = msg.getQueueId();
log.info("消费消息,队列id:" +queueId + ",msg:" + body);
//做业务逻辑操作 TODO
// log.info("消费消息:" + "topic=" + topic + ", tags=" + tags + ", keys=" + keys + ", msg=" + body);
return ConsumeOrderlyStatus.SUCCESS;
} catch (UnsupportedEncodingException e) {
log.error("消费异常" + e.getMessage());
//如果重试2次不成功，则记录，人工介入
if(times >= 2){
log.error("重试次数大于2，记录数据库，发短信通知开发人员或者运营人员");
//TODO 记录数据库，发短信通知开发人员或者运营人员
//告诉broker，消息成功
return ConsumeOrderlyStatus.SUCCESS;
}
return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
}
});
consumer.start();
log.info("consumer start ...");
}
}
事务消息示例
- MqTransactionCallback.java
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
/**
* @author zhang3
* description 具体业务
*/
public interface MqTransactionCallback {
/**
* 执行实际业务
* @param msg mq消息
* @param arg 传递的参数
* @return 处理结果 1.成功, 2.回滚, 3.未知
*/
int executeLocalBusiness(Message msg, Object arg);
/**
* 检查实际业务是否完成
* @param msg mq消息
* @return 处理结果 1.成功, 2.回滚, 3.未知
*/
int checkLocalBusiness(MessageExt msg);
}
- MqTransactionListener.java
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.producer.LocalTransactionState;
import org.apache.rocketmq.client.producer.TransactionListener;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
/**
* @author zhang3
* description TODO
*/
@Slf4j
public abstract class MqTransactionListener implements TransactionListener, MqTransactionCallback {
@Override
public LocalTransactionState executeLocalTransaction(Message msg, Object arg) {
log.info("====executeLocalTransaction=======");
String body = new String(msg.getBody());
String key = msg.getKeys();
String transactionId = msg.getTransactionId();
log.info("transactionId="+transactionId+", key="+key+", body="+body);
// 执行本地事务begin
int status = executeLocalBusiness(msg, arg);
// 执行本地事务end
//二次确认消息，然后消费者可以消费
if(status == 1){
return LocalTransactionState.COMMIT_MESSAGE;
}
//回滚消息，broker端会删除半消息
if(status == 2){
return LocalTransactionState.ROLLBACK_MESSAGE;
}
//broker端会进行回查消息，再或者什么都不响应
if(status == 3){
return LocalTransactionState.UNKNOW;
}
return null;
}
@Override
public LocalTransactionState checkLocalTransaction(MessageExt msg) {
log.info("====checkLocalTransaction=======");
String body = new String(msg.getBody());
String key = msg.getKeys();
String transactionId = msg.getTransactionId();
log.info("transactionId="+transactionId+", key="+key+", body="+body);
//要么commit 要么rollback
int status = checkLocalBusiness(msg);
if (status == 1) {
return LocalTransactionState.COMMIT_MESSAGE;
} else if (status == 2) {
return LocalTransactionState.ROLLBACK_MESSAGE;
} else {
return LocalTransactionState.UNKNOW;
}
}
}
- DemoTransactionListener.java
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;
import java.io.UnsupportedEncodingException;
/**
* @author zhang3
* description 分布式事务-处理具体业务
*/
@Slf4j
public class DemoTransactionListener extends MqTransactionListener {
@Override
public int executeLocalBusiness(Message msg, Object arg) {
int result = 2;
try {
//TODO 处理相关的业务
String body = new String(msg.getBody(), "utf-8");
if (body.contains("a")) {
result = 1;
}
} catch (UnsupportedEncodingException e) {
result = 2;
log.error(e.getMessage(), e);
}
if (result == 1) {
log.info("处理相关的业务-处理成功!");
return 1;
} else {
return result;
}
}
@Override
public int checkLocalBusiness(MessageExt msg) {
int result = 2;
try {
//TODO 检查相关的业务
String body = new String(msg.getBody(), "utf-8");
if (body.contains("a")) {
result = 1;
}
} catch (UnsupportedEncodingException e) {
result = 2;
log.error(e.getMessage(), e);
}
if (result == 1) {
log.info("检查相关的业务-执行完成");
return 1;
} else {
return result;
}
}
}
- MqTransactionProducer.java
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.TransactionListener;
import org.apache.rocketmq.client.producer.TransactionMQProducer;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.concurrent.*;
/**
* @author zhang3
* description MQ分布式事务
*/
@Data
@Slf4j
public class MqTransactionProducer {
public MqTransactionProducer(TransactionListener transactionListener) {
this.transactionListener = transactionListener;
}
/**
* 事务监听器
*/
private TransactionListener transactionListener;
private TransactionMQProducer producer = null;
//一般自定义线程池的时候，需要给线程加个名称
private ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS,
new ArrayBlockingQueue(2000), new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("client-transaction-msg-check-thread");
return thread;
}
});
/**
* 对象在使用之前必须要调用一次，只能初始化一次
*/
@PostConstruct
public void start(){
try {
producer = new TransactionMQProducer(JmsConfig.TRANSACTION_PRODUCER_GROUP);
producer.setNamesrvAddr(JmsConfig.NAME_SERVER);
producer.setTransactionListener(transactionListener);
producer.setExecutorService(executorService);
//指定NameServer地址，多个地址以 ; 隔开
producer.start();
} catch (MQClientException e) {
e.printStackTrace();
}
}
/**
* 一般在应用上下文，使用上下文监听器，进行关闭
*/
@PreDestroy
public void shutdown(){
this.producer.shutdown();
}
}
- MqTransactionConsumer.java
import lombok.extern.slf4j.Slf4j;
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import javax.annotation.PostConstruct;
import java.util.List;
/**
* @author zhang3
* description TODO
*/
@Slf4j
public class MqTransactionConsumer {
private DefaultMQPushConsumer consumer;
@PostConstruct
public void ini() throws MQClientException {
consumer = new DefaultMQPushConsumer(JmsConfig.CONSUMER_TRANSACTION_GROUP);
consumer.setNamesrvAddr(JmsConfig.NAME_SERVER);
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
//默认是集群方式，可以更改为广播，但是广播方式不支持重试
consumer.setMessageModel(MessageModel.CLUSTERING);
consumer.subscribe(JmsConfig.DEFAULT_TRANSACTION_TOPIC,"*");
consumer.registerMessageListener( new MessageListenerConcurrently() {
@Override
public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) {
MessageExt msg = msgs.get(0);
try {
String topic = msg.getTopic();
String body = new String(msg.getBody(), "utf-8");
String tags = msg.getTags();
String keys = msg.getKeys();
log.info("消费消息:" + "topic=" + topic + ", tags=" + tags + ", keys=" + keys + ", msg=" + body);
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
} catch (Exception e) {
System.out.println("消费异常");
e.printStackTrace();
return ConsumeConcurrentlyStatus.RECONSUME_LATER;
}
}
});
consumer.start();
System.out.println("consumer start ...");
}
}
测试代码
import org._33web.cloud.rocket.mq.jms.JmsConfig;
import org._33web.cloud.rocket.mq.jms.orderly.MqOrderlyProducer;
import org._33web.cloud.rocket.mq.jms.simple.MqProducer;
import org._33web.cloud.rocket.mq.jms.transaction.MqTransactionProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
@RestController
public class DemoController {
@Autowired
private MqProducer mqProducer;
@RequestMapping("/test/send/msg")
public Object sendMsg(String text) {
Message message = JmsConfig.Utils.buildDefaultMessage ("hello rocket mq = "+text);
SendResult sendResult = mqProducer.sendDelayMsg(message);
System.out.println(sendResult);
return new HashMap<>(5);
}
@Autowired
private MqOrderlyProducer mqOrderlyProducer;
/**
* 测试发送顺序消息，以及消费顺序消息
* @param text 消息内容
* @return 返回结果
*/
@RequestMapping("/test/send/orderly/msg")
public Object sendOrderlyMsg(String text) {
for (int i = 0; i < 10; i++) {
int num = i % 4;
Message message = JmsConfig.Utils.buildOrderlyMessage ("hello rocket mq = "+text + i + "," + num);
SendResult sendResult = mqOrderlyProducer.sendQueueSelectorMsg(message, num);
System.out.println(sendResult);
}
return new HashMap<>(5);
}
@Autowired
private MqTransactionProducer mqTransactionProducer;
/**
* 测试发送分布式事务消息，并且进行消费
* @param text 消息内容
* @param otherParam 相关业务参数
* @return 返回结果
* @throws Exception Mq异常
*/
@RequestMapping("/test/send/transaction/msg")
public Object sendTransactionMsg(String text, String otherParam ) throws Exception {
Message message = JmsConfig.Utils.buildTransactionMessage(text);
SendResult sendResult = mqTransactionProducer.getProducer().
sendMessageInTransaction(message, otherParam);
System.out.println(sendResult);
return new HashMap<>(5);
}
}
