自定义对rocketmq生产者以及消费者的封装

一. 生产者抽象类

package com.canxiusi.common.service.customize_mq.producer;

import com.canxiusi.common.service.customize_mq.constant.MqConstant;
import com.canxiusi.common.service.exception.SystemException;
import com.canxiusi.common.service.exception.enums.CommonExceptionEnum;
import com.canxiusi.common.service.log4j2.spi.Logger;
import com.canxiusi.common.service.log4j2.spi.LoggerFactory;
import com.canxiusi.common.service.utils.LogUtils;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.exception.RemotingException;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

/**
 * @author canxiusi.yan
 * @description AbstractMqProducer 抽象SysLog生产者, 父类定义在common模块
 * @date 2022/5/8 19:46
 */
public abstract class AbstractSysLogProducer {

    /**
     * 日志
     */
    private static final Logger logger = LoggerFactory.getLogger(AbstractSysLogProducer.class);

    private DefaultMQProducer sysLogProducer;

    /**
     * 初始化mq生产者
     */
    @PostConstruct
    public void initProducer() {
        sysLogProducer = new DefaultMQProducer(getProducerGroup());
        sysLogProducer.setNamesrvAddr(getNameServer());
        sysLogProducer.setInstanceName(getInstanceName());
        try {
            sysLogProducer.start();
            logger.info(LogUtils.format("[SysLog生产者初始化完成], producer=<{0}>", sysLogProducer));
        } catch (MQClientException e) {
            logger.error("[SysLog生产者初始化失败]", e);
            System.exit(-1);
        }
    }

    @PreDestroy
    public void closeProducer() {
        sysLogProducer.shutdown();
    }

    /**
     * 发送日志单向消息
     *
     * @param bytes
     */
    public void sendMsgByteOneway(byte[] bytes) {
        if (bytes.length > MqConstant.MAX_MB) {
            logger.error(LogUtils.format("[发送消息过大, 无法发送!], msg=<{0}>", new String(bytes)));
            throw new SystemException(CommonExceptionEnum.PARAM_ERROR, "[sendMsgByteOneway] 发送消息大于4MB!");
        }
        Message message = new Message(getTopic(), getTag(), bytes);
        try {
            sysLogProducer.sendOneway(message);
        } catch (MQClientException | RemotingException | InterruptedException e) {
            logger.error(LogUtils.format("线程<{0}>, 生产者=<{1}>发送单向消息出错",
                    Thread.currentThread().getName(), this.getClass().getSimpleName()), e);
        }
    }

    /**
     * 获取mq地址
     *
     * @return
     */
    protected abstract String getNameServer();

    /**
     * 获取日志生产者
     *
     * @return
     */
    protected abstract String getProducerGroup();

    /**
     * 获取实例名
     *
     * @return
     */
    protected abstract String getInstanceName();

    /**
     * 获取topic
     *
     * @return
     */
    protected abstract String getTopic();

    /**
     * 获取tags
     *
     * @return
     */
    protected abstract String getTag();

}

二. 生产者子类实现

package com.canxiusi.web.online.producer;

import com.canxiusi.common.service.customize_mq.config.SysLogMqConfig;
import com.canxiusi.common.service.customize_mq.producer.AbstractSysLogProducer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * @author canxiusi.yan
 * @description SysLogProducerInstance 日志落入DB消费者
 * @date 2022/5/10 14:52
 */
@Component
public class SysLogProducer extends AbstractSysLogProducer {

    @Autowired
    private SysLogMqConfig sysLogMqConfig;

    @Override
    protected String getNameServer() {
        return sysLogMqConfig.getNameServerAddress();
    }

    @Override
    protected String getProducerGroup() {
        return sysLogMqConfig.getProducerGroup();
    }

    @Override
    protected String getInstanceName() {
        return sysLogMqConfig.getProducerInstance();
    }

    @Override
    protected String getTopic() {
        return sysLogMqConfig.getTopic();
    }

    @Override
    protected String getTag() {
        return sysLogMqConfig.getTag();
    }

}

三. 配置文件bean

package com.canxiusi.common.service.customize_mq.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;

/**
 * @author canxiusi.yan
 * @description SysLogMqConfig 日志mq配置bean
 * @date 2022/5/15 11:08
 */
@Data
@Configuration
@PropertySource("classpath:syslogmq.properties")
@ConfigurationProperties(prefix = "syslog")
public class SysLogMqConfig {

    private String nameServerAddress;

    private String producerGroup;

    private String producerInstance;

    private String consumerGroup;

    private String consumerDbGroup;

    private String consumerMongoGroup;

    private String consumerDbInstance;

    private String consumerMongoInstance;

    private String topic;

    private String tag;

    private String dbTopic;

    private String mongoTag;

}

四. 消费者抽象实现

package com.canxiusi.common.service.customize_mq.consumer;

import com.canxiusi.common.service.entity.logs.SysOperateLog;
import com.canxiusi.common.service.log4j2.spi.Logger;
import com.canxiusi.common.service.log4j2.spi.LoggerFactory;
import com.canxiusi.common.service.utils.JsonUtil;
import com.canxiusi.common.service.utils.LogUtils;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.springframework.boot.context.event.ApplicationPreparedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.util.CollectionUtils;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

/**
 * @author canxiusi.yan
 * @description AbstractProducer 抽象SysLog抽象父类
 * @date 2022/5/10 10:03
 */
public abstract class AbstractSysLogConsumer {

    /**
     * 子类通用日志
     */
    protected static final Logger logger = LoggerFactory.getLogger(AbstractSysLogConsumer.class);

    private DefaultMQPushConsumer consumer;

    /**
     * 初始化mq消费者
     */
    @PostConstruct
    public void initConsumer() {
        consumer = new DefaultMQPushConsumer(getConsumerGroup());
        consumer.setNamesrvAddr(getNameServer());
        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
        consumer.setMessageModel(MessageModel.BROADCASTING);
        consumer.setInstanceName(getInstanceName());
        consumer.setPullInterval(2000);
        consumer.setPullBatchSize(500);
        try {
            consumer.subscribe(getTopic(), getTag());
        } catch (MQClientException e) {
            logger.error("[SysLog消费者初始化异常]", e);
            System.exit(-1);
        }
        consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
            if (CollectionUtils.isEmpty(msgs)) {
                logger.info(LogUtils.format("[消息列表为空], msgs=<{0}>", msgs));
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
            for (MessageExt msg : msgs) {
                try {
                    handlerMsg(msg);
                } catch (Exception e) {
                    logger.error("[日志消费处理异常]", e);
                }
            }
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        });
        logger.info(LogUtils.format("[SysLog消费者初始化完成], consumer=<{0}>", consumer));
    }

    /**
     * 启动消费者
     *
     * @param event
     */
    @EventListener(ApplicationPreparedEvent.class)
    public void startSysLogConsumer(ApplicationPreparedEvent event) {
        try {
            consumer.start();
        } catch (MQClientException e) {
            logger.error(LogUtils.format("[SysLog消费者start异常], nameServerAddress={}", getNameServer()), e);
            System.exit(-1);
        }
    }

    @PreDestroy
    public void closeConsumer() {
        consumer.shutdown();
    }

    /**
     * 调用子类去消费消息
     *
     * @param msg
     */
    protected final void handlerMsg(MessageExt msg) {
        SysOperateLog sysOperateLog;
        try {
            sysOperateLog = JsonUtil.readValue(msg.getBody(), SysOperateLog.class);
        } catch (Exception e) {
            logger.error("[字节消息转换出错]", e);
            throw e;
        }
        handlerLogs(sysOperateLog);
    }

    /**
     * 子类处理消息
     *
     * @param sysOperateLog
     */
    protected abstract void handlerLogs(SysOperateLog sysOperateLog);

    /**
     * 获取mq地址
     *
     * @return
     */
    protected abstract String getNameServer();

    /**
     * 获取系统日志消费者组, 消费者也可以在同一分组, 子类重写获取不同的分组
     *
     * @return
     */
    protected abstract String getConsumerGroup();

    /**
     * 获取消费者实例名称
     *
     * @return
     */
    protected abstract String getInstanceName();

    /**
     * 获取消费者topic
     *
     * @return
     */
    protected abstract String getTopic();

    /**
     * 获取消费者tag
     *
     * @return
     */
    protected abstract String getTag();

}

五. 消费者子类实现:

5.1 模拟mysql消费者

package com.canxiusi.web.service.consumer;

import com.canxiusi.common.service.customize_mq.config.SysLogMqConfig;
import com.canxiusi.common.service.customize_mq.consumer.AbstractSysLogConsumer;
import com.canxiusi.common.service.entity.logs.SysOperateLog;
import com.canxiusi.common.service.log4j2.spi.Logger;
import com.canxiusi.common.service.log4j2.spi.LoggerFactory;
import com.canxiusi.common.service.utils.LogUtils;
import com.canxiusi.web.service.mapper.operate.SysOperateLogMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * @author canxiusi.yan
 * @description SysLogsConsumer 日志消费者, 监听系统日志
 * @date 2022/5/10 11:07
 */
@Component
public class SysLogDbConsumer extends AbstractSysLogConsumer {

    /**
     * 日志
     */
    private static final Logger logger = LoggerFactory.getLogger(SysLogDbConsumer.class);

    @Autowired
    private SysLogMqConfig sysLogMqConfig;

    @Autowired
    private SysOperateLogMapper operateLogMapper;

    @Override
    protected void handlerLogs(SysOperateLog sysOperateLog) {
        logger.info(LogUtils.format("当前线程<{0}>, <{1}>消费了消息, sysLog=<{2}>",
                Thread.currentThread().getName(), this.getClass().getSimpleName(), sysOperateLog));
        operateLogMapper.insertSelective(sysOperateLog);
    }

    @Override
    protected String getNameServer() {
        return sysLogMqConfig.getNameServerAddress();
    }

    @Override
    protected String getConsumerGroup() {
        return sysLogMqConfig.getConsumerGroup();
    }

    @Override
    protected String getInstanceName() {
        return sysLogMqConfig.getConsumerDbInstance();
    }

    @Override
    protected String getTopic() {
        return sysLogMqConfig.getTopic();
    }

    @Override
    protected String getTag() {
        return sysLogMqConfig.getTag();
    }

}

5.2 模拟mongo消费者

package com.canxiusi.web.service.consumer;

import com.canxiusi.common.service.customize_mq.config.SysLogMqConfig;
import com.canxiusi.common.service.customize_mq.consumer.AbstractSysLogConsumer;
import com.canxiusi.common.service.entity.logs.SysOperateLog;
import com.canxiusi.common.service.log4j2.spi.Logger;
import com.canxiusi.common.service.log4j2.spi.LoggerFactory;
import com.canxiusi.common.service.utils.LogUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * @author canxiusi.yan
 * @description SysLogMongoConsumer
 * @date 2022/5/10 20:47
 */
@Component
public class SysLogMongoConsumer extends AbstractSysLogConsumer {

    private static final Logger logger = LoggerFactory.getLogger(SysLogMongoConsumer.class);

    @Autowired
    private SysLogMqConfig sysLogMqConfig;

    @Override
    protected void handlerLogs(SysOperateLog sysOperateLog) {
        logger.info(LogUtils.format("当前线程<{0}>, <{1}>消费了消息, sysLog=<{2}>",
                Thread.currentThread().getName(), this.getClass().getSimpleName(), sysOperateLog));
    }

    @Override
    protected String getNameServer() {
        return sysLogMqConfig.getNameServerAddress();
    }

    @Override
    protected String getConsumerGroup() {
        return sysLogMqConfig.getConsumerGroup();
    }

    @Override
    protected String getInstanceName() {
        return sysLogMqConfig.getConsumerMongoInstance();
    }

    /**
     * 一个topic, 用tags来区分
     *
     * @return
     */
    @Override
    protected String getTopic() {
        return sysLogMqConfig.getTopic();
    }

    @Override
    protected String getTag() {
        return sysLogMqConfig.getTag();
    }

}

六. 具体发送消息实例

6.1 默认的异步发送

        如果使用 RocketMQ 默认的异步 Producer, 首先我们需要使用 producer 的 send() 方法, 接收一个 sendCallback 回调

sysLogProducer.send(message, new SendCallback() {
                @Override
                public void onSuccess(SendResult sendResult) {

                }

                @Override
                public void onException(Throwable e) {

                }
            });

        查看源码发现, 在发送消息的时候使用了线程池开启了异步, 如果自己没有不定义线程池的话, mq会使用默认的线程池(不适用), 所以需要一定要根据自己业务场景, 定义线程池, 然后通过下面的方法把线程池设置进去

public void setAsyncSenderExecutor(final ExecutorService asyncSenderExecutor) {
        this.defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor);
    }

public void send(final Message msg, final SendCallback sendCallback, final long timeout)
        throws MQClientException, RemotingException, InterruptedException {
        final long beginStartTime = System.currentTimeMillis();
        ExecutorService executor = this.getAsyncSenderExecutor();
        try {
            executor.submit(new Runnable() {
                @Override
                public void run() {
                    long costTime = System.currentTimeMillis() - beginStartTime;
                    if (timeout > costTime) {
                        try {
                            sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime);
                        } catch (Exception e) {
                            sendCallback.onException(e);
                        }
                    } else {
                        sendCallback.onException(
                            new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout"));
                    }
                }

            });
        } catch (RejectedExecutionException e) {
            throw new MQClientException("executor rejected ", e);
        }

    }

6.2 自定义异步发送

        上面的 send() 方法的效率不高, 因为他需要一个 sendCallback 回调, 对于日志场景, 我们不关心回调, 所以可以使用效率更高的 sendOnway() 方法, 但是查看源码发现, 这个方法没有开启线程实现异步, 针对这一点可以实现自定义异步发送

        我这里是配合spring事件监听机制以及策略模式又做了一层解耦合, 把消息的构建和发送消息的逻辑分离开, 而且使用了异步, 在调用 doSpecificHandler() 方法之前就开启了异步, 这样的话就实现了构建字节消息, 发送消息的异步, 如果使用 6.1 的默认异步方式, 构建字节消息并不是在异步中实现的

        首先是使用线程池开启异步

    @Override
    public void asyncExecuteIgnoreResult(String routingKey, FlowContext context) {
        Strategy strategy = FlowExecutor.getStrategy(routingKey);
        try {
            threadPoolExecutor.execute(() -> strategy.doSpecificHandler(null, context));
        } catch (Throwable e) {
            logger.error(LogUtils.format("[asyncExecuteIgnoreResult 线程调度任务出现错误或异常], routingKey={0}, obj={1}",
                    routingKey, e.toString()), e);
        }
    }

        然后使用策略模式, 找到具体的策略实现类, 构建字节消息, 装配 sysLogProducer 对象, 调用 sendMsgByteOneway () 方法发送单向消息

/**
 * @author canxiusi.yan
 * @description SysLogsEventSender 日志消息发送器
 * @date 2022/5/3 10:03
 */
@Component
@StrategyAnno(routingKey = EventConstant.SYS_LOG_SENDER, weight = "10010")
public class SysLogEventSender implements Strategy {

    /**
     * 日志
     */
    private static final Logger logger = LoggerFactory.getLogger(SysLogEventSender.class);

    @Autowired
    private SysLogProducer sysLogProducer;

    @Override
    public void doSpecificHandler(Flow flow, FlowContext flowContext) {
        long now = System.currentTimeMillis();
        // 获取不为null的sysLog
        SysOperateLog operateLog = Optional.ofNullable(
                        ExecuteDataHelper.getParam(flowContext.getGlobalMap(), CommonMapParam.SYS_LOG_LOG))
                .orElse(new SysOperateLog());
        // 构建字节消息
        byte[] bytes = JsonUtil.toJsonAsBytes(operateLog);
        // 发送mq消息
        sysLogProducer.sendMsgByteOneway(bytes);
        long end = System.currentTimeMillis();
        logger.info(LogUtils.format("当前线程<{0}>, <{1}>发送消息, flowContext=<{2}>, 耗时=<{3}>毫秒",
                Thread.currentThread().getName(),
                this.getClass().getSimpleName(),
                flowContext.getGlobalMap(),
                end - now));
    }
    
}

七. 运行测试

7.1 生产者发送消息测试

        首先是生产者发送消息(消息内容可以忽略, 模拟的), 这里我自定了线程工厂和线程名称, 可以看到线程已经不再是 tomcat 默认分配的 http-nio-端口号, 而是我使用线程池开启的新线程

---

7.2 消费者消费消息测试 

        消费者消费消息情况, 可以看到两个消费者都消费了消息, 并且是异步消费, 但是这个线程是由 rocketmq 内部产生的

八. 登录控制台查看

        这里模拟的场景是发送一条消息被2个消费者消费