- Client端
- Producer Group 一类Producer的集合名称,这类Producer通常发送一类消息,且发送逻辑一致Consumer
Group 一类Consumer的集合名称,这类Consumer通常消费一类消息,且消费逻辑一致。
- Server端
- Broker 消息中转角色,负责存储消息,转发消息,这里就是RocketMQ ServerTopic
消息的主题,用于定义并在服务端配置,消费者可以按照主题进行订阅,也就是消息分类,通常一个系统一个Topic
- Message 在生产者、消费者、服务器之间传递的消息,一个message必须属于一个Topic
消息是要传递的信息。邮件中必须包含一个主题,该主题可以解释为要发送给您的信的地址。消息还可能具有可选标签和额外的键值对。
- 您可以为消息设置业务密钥,然后在代理服务器上查找消息以在开发过程中诊断问题。
- Namesrver
一个无状态的名称服务,可以集群部署,每一个broker启动的时候都会向名称服务器注册,主要是接收broker的注册,接收客户端的路由请求并返回路由信息Offset
偏移量,消费者拉取消息时需要知道上一次消费到了什么位置, 这一次从哪里开始Partition
分区,Topic物理上的分组,一个Topic可以分为多个分区,每个分区是一一个有序的队列。
- 分区中的每条消息都会给分配一个有序的ID,也就是偏移量,保证了顺序,消费的正确性Tag
用于对消息进行过滤,理解为message的标记,同一业务不同目的的message可以用相同的topic但是 可以用不同的tag来区分key
消息的KEY字段是为了唯- -表示消息的,方便查问题,不是说必须设置,只是说设置为了方便开发和运维定位问题。
package com.programb.springboot.rocketmq.starter.common;
import com.alibaba.fastjson.JSON;
import com.programb.springboot.rocketmq.starter.constants.RocketMqContent;
import com.programb.springboot.rocketmq.starter.constants.RocketMqTag;
import com.programb.springboot.rocketmq.starter.constants.RocketMqTopic;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.message.MessageExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
import java.util.Set;
public abstract class AbstractRocketMqConsumer
<Topic extends RocketMqTopic, Tag extends RocketMqTag, Content extends RocketMqContent>
implements MessageListenerConcurrently
{
protected final Logger logger = LoggerFactory.getLogger(getClass());
protected Class<Topic> topicClazz;
protected Class<Tag> tagClazz;
protected Class<Content> contentClazz;
public abstract Map<String, Set<String>> subscribeTopicTags();
public abstract boolean handle(String topic, String tag, Content content, MessageExt msg);
@PostConstruct
public void init() {
Class<? extends AbstractRocketMqConsumer> parentClazz = this.getClass();
Type genType = parentClazz.getGenericSuperclass();
Type[] types = ((ParameterizedType) genType).getActualTypeArguments();
topicClazz = (Class<Topic>) types[0];
contentClazz = (Class<Content>) types[2];
logger.info("topicClazz:{}, contentClazz:{}", topicClazz, contentClazz);
}
public Class<?> getModelClass(Class modelClass, int index) {
Type genType = this.getClass().getGenericSuperclass();
Type[] params = ((ParameterizedType) genType).getActualTypeArguments();
if (params.length - 1 < index) {
modelClass = null;
} else {
modelClass = (Class) params[index];
}
return modelClass;
}
@Override
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
MessageExt msg = msgs.get(0);
byte[] body = msg.getBody();
String topic = msg.getTopic();
String tags = msg.getTags();
long bornTimestamp = msg.getBornTimestamp();
long currentTimeMillis = System.currentTimeMillis();
long timeElapsedFromStoreInMqToReceiveMsg = currentTimeMillis - bornTimestamp;
ConsumeConcurrentlyStatus consumeConcurrentlyStatus = ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
if (timeElapsedFromStoreInMqToReceiveMsg >= 300000) {
logger.warn("msg:{} is invalid, it was born {}s ago", msg, timeElapsedFromStoreInMqToReceiveMsg / 1000);
return consumeConcurrentlyStatus;
}
try {
consumeConcurrentlyStatus = handle(topic, tags, parseMsg(body, contentClazz), msg) ? ConsumeConcurrentlyStatus.CONSUME_SUCCESS : ConsumeConcurrentlyStatus.RECONSUME_LATER;
} catch (Throwable t) {
logger.warn("mq handler error, msg info:{}", msg, t);
}
return consumeConcurrentlyStatus;
}
private String getBodyString(byte[] body) {
String bodyString = null;
if (null != body) {
try {
bodyString = new String(body, "UTF-8");
} catch (UnsupportedEncodingException e) {
logger.error("can not parse to string with UTF-8", e);
}
}
return bodyString;
}
private <T> T parseMsg(byte[] body, Class<? extends RocketMqContent> clazz){
T t = null;
if (body != null) {
try {
t = JSON.parseObject(body, clazz);
} catch (Exception e) {
logger.error("can not parse to object", e);
}
}
return t;
}
}
package com.programb.springboot.rocketmq.starter.config;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.CollectionUtils;
import com.programb.springboot.rocketmq.starter.common.AbstractRocketMqConsumer;
import javax.annotation.Resource;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Configuration
@ConditionalOnClass({ DefaultMQPushConsumer.class })
@EnableConfigurationProperties(RocketMqProperties.class)
public class RocketMqAutoConfiguration {
private final static Logger LOGGER = LoggerFactory.getLogger(RocketMqAutoConfiguration.class);
@Resource
private RocketMqProperties rocketMqProperties;
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(value = AbstractRocketMqConsumer.class)
public DefaultMQPushConsumer defaultMQPushConsumer(List<AbstractRocketMqConsumer> messageListeners) {
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(rocketMqProperties.getConsumerGroupName());
if (rocketMqProperties.getConsumeThreadMin() != null) {
consumer.setConsumeThreadMin(rocketMqProperties.getConsumeThreadMin());
}
if (rocketMqProperties.getConsumeThreadMax() != null) {
consumer.setConsumeThreadMax(rocketMqProperties.getConsumeThreadMax());
}
if (rocketMqProperties.getMessageModel() != null) {
consumer.setMessageModel(MessageModel.valueOf(rocketMqProperties.getMessageModel()));
}
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
consumer.setNamesrvAddr(rocketMqProperties.getNameServer());
messageListeners.forEach(messageListener -> {
Map<String, Set<String>> subscribeTopicTags = messageListener.subscribeTopicTags();
subscribeTopicTags.entrySet().forEach(e -> {
try {
String rocketMqTopic = e.getKey();
Set<String> rocketMqTags = e.getValue();
if (CollectionUtils.isEmpty(rocketMqTags)) {
consumer.subscribe(rocketMqTopic, "*");
} else {
String tags = StringUtils.join(rocketMqTags, " || ");
consumer.subscribe(rocketMqTopic, tags);
LOGGER.info("subscribe, topic:{}, tags:{}", rocketMqTopic, tags);
}
} catch (MQClientException ex) {
LOGGER.error("consumer subscribe error", ex);
}
});
consumer.registerMessageListener(messageListener);
});
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
LOGGER.info("consumer shutdown");
consumer.shutdown();
LOGGER.info("consumer has shutdown");
}));
try {
consumer.start();
LOGGER.info("rocketmq consumer started, nameserver:{}, group:{}", rocketMqProperties.getNameServer(),
rocketMqProperties.getConsumerGroupName());
} catch (MQClientException e) {
LOGGER.error("consumer start error, nameserver:{}, group:{}", rocketMqProperties.getNameServer(),
rocketMqProperties.getConsumerGroupName(), e);
}
return consumer;
}
}
package com.programb.springboot.rocketmq.starter.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = RocketMqProperties.ROCKETMQ_PREFIX)
public class RocketMqProperties {
static final String ROCKETMQ_PREFIX = "spring.rocketmq";
private String nameServer;
private String producerGroupName;
private String consumerGroupName;
private Integer consumeThreadMin;
private Integer consumeThreadMax;
private String messageModel;
public String getNameServer() {
return nameServer;
}
public void setNameServer(String nameServer) {
this.nameServer = nameServer;
}
public String getProducerGroupName() {
return producerGroupName;
}
public void setProducerGroupName(String producerGroupName) {
this.producerGroupName = producerGroupName;
}
public String getConsumerGroupName() {
return consumerGroupName;
}
public void setConsumerGroupName(String consumerGroupName) {
this.consumerGroupName = consumerGroupName;
}
public Integer getConsumeThreadMin() {
return consumeThreadMin;
}
public void setConsumeThreadMin(Integer consumeThreadMin) {
this.consumeThreadMin = consumeThreadMin;
}
public Integer getConsumeThreadMax() {
return consumeThreadMax;
}
public void setConsumeThreadMax(Integer consumeThreadMax) {
this.consumeThreadMax = consumeThreadMax;
}
public String getMessageModel() {
return messageModel;
}
public void setMessageModel(String messageModel) {
this.messageModel = messageModel;
}
}
package com.programb.springboot.rocketmq.starter.constants;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import java.io.Serializable;
public class RocketMqContent implements Serializable {
private static final long serialVersionUID = 1L;
@Override
public String toString() {
return JSON.toJSONString(this, SerializerFeature.NotWriteDefaultValue);
}
}
package com.programb.springboot.rocketmq.starter.constants;
public interface RocketMqTag {
String getTag();
}
package com.programb.springboot.rocketmq.starter.constants;
public interface RocketMqTopic {
String getTopic();
}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>spring-boot-programb-examples</artifactId>
<groupId>com.programb</groupId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>spring-boot-rocketmq-starter</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>