RocketMq实用

1. rocketmq 生产者

package com.becom.qoe.qoeservice.service.rocketmq.impl;

import java.io.UnsupportedEncodingException;

import javax.annotation.PostConstruct;

import org.apache.commons.lang3.time.StopWatch;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import com.becom.qoe.qoeservice.entity.stuscorestatistics.StuNormScore;
import com.becom.qoe.qoeservice.service.rocketmq.IRocketMqProducer;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Service
public class RocketMqProducer implements IRocketMqProducer<StuNormScore>{
	@Value("${apache.rocketmq.producer.producerGroup}")
	private String producerGroup;
	
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	
	private DefaultMQProducer producer;
	
	@PostConstruct
	public void defaultMQProducer() {
		// 生产者的组名
		producer = new DefaultMQProducer(producerGroup);
		// 指定NameServer地址，多个地址以 ; 隔开
		producer.setNamesrvAddr(namesrvAddr);
		producer.setVipChannelEnabled(false);
		try {
			producer.start();
			log.info("MQ：启动启动生产者");
		} catch (MQClientException e) {
			e.printStackTrace();
		}
	}
	
	@Override
	public boolean send(StuNormScore om) {
		JSONObject json = JSONObject.fromObject(om);
		Message message;
		StopWatch stop = new StopWatch();
		stop.start();
		boolean stat = false;
		try {
			message = new Message(topic, tag, json.toString().getBytes(RemotingHelper.DEFAULT_CHARSET));
			SendResult result = producer.send(message);
			log.info("MQ 消息生产者发送消息状态: "+ result.getSendStatus());
			if (SendStatus.SEND_OK.equals(result.getSendStatus())) {
				stat = true;
			}
		} catch (UnsupportedEncodingException | MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
			log.error(e.getMessage(), e);
		}
		stop.stop();
		return stat;
	}

}

2. rocketmq 消费者

消费者类型1：PushConsumer

package com.becom.qoe.asyncproccess.rocketmq;

import java.util.List;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;

import com.becom.qoe.asyncproccess.entity.StuNormScore;
import com.becom.qoe.asyncproccess.service.MainExecuteStatistic;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Component
public class RocketMqConsumer implements CommandLineRunner{

	@Value("${apache.rocketmq.consumer.ConsumerGroup}")
	private String consumerGroupName;
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	@Autowired
	private MainExecuteStatistic mainExecuteStatistic;
	/**
	 * MQ 消息消费端
	 * 消费策略：
	 * CONSUME_FROM_LAST_OFFSET 默认策略，从该队列最尾开始消费，即跳过历史消息
	 * CONSUME_FROM_FIRST_OFFSET 从队列最开始开始消费，即历史消息（还储存在broker的）全部消费一遍
	 * CONSUME_FROM_TIMESTAMP 从某个时间点开始消费，和setConsumeTimestamp()配合使用，默认是半个小时以前
	 * 消费模式：
	 * CLUSTERING：集群，默认
	 * 同一个Group里每个consumer只消费订阅消息的一部分内容，也就是同一groupName,所有消费的内容加起来才是订阅topic内容的整体，达到负载均衡的目的
	 * BROADCASTING:广播模式
	 * 同一个Group里每个consumer都能消费到所订阅topic的全部消息，也就是一个消息会被分发多次，被多个consumer消费
	 * 广播消息只发送一次，没有重试
	 * 返回消费状态：
	 * CONSUME_SUCCESS 消费成功  
	 * RECONSUME_LATER 消费失败，需要稍后重新消费
	 * 重试机制(consumer),仅限于CLUSTERING模式
	 * 1.exception的情况，一般重复16次 10s、30s、1分钟、2分钟、3分钟等等  获取重试次数：msgs.get(0).getReconsumeTimes()
	 * 2.超时的情况，这种情况MQ会无限制的发送给消费端 就是由于网络的情况，MQ发送数据之后，Consumer端并没有收到导致超时。也就是消费端没有给我返回
	 * return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;这样的就认为没有到达Consumer端
	 */
	public void messageListener() {
		log.info("MQ：启动消费者");
		DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroupName);

		consumer.setNamesrvAddr(namesrvAddr);
		try {
			// 订阅PushTopic下Tag为push的消息,都订阅消息
			consumer.subscribe(topic, tag);
			// 程序第一次启动从消息队列头获取数据
			consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
			consumer.setMessageModel(MessageModel.CLUSTERING);
			// 可以修改每次消费消息的数量，默认设置是每次消费一条
			consumer.setConsumeMessageBatchMaxSize(1);
			// 在此监听中消费信息，并返回消费的状态信息
			/*consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> {
				// 会把不同的消息分别放置到不同的队列中
				for (Message msg : msgs) {
					StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(new String(msg.getBody())), StuNormScore.class);
					log.info("rocketMQ client 成功获取消息" + msg);
					mainExecuteStatistic.executeStatistic(stuNormScore);
				}
				return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
			});*/
			consumer.registerMessageListener(new MessageListenerConcurrently() {
				@Override
				public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
						ConsumeConcurrentlyContext context) {
					
					int index = 0;
					try {
						for (; index < msgs.size(); index++) {
							log.info("MQ 消费者 获取消息重试次数" + msgs.get(index).getReconsumeTimes());
							MessageExt msg = msgs.get(index);
							String messageBody = new String(msg.getBody());
							log.info("MQ 消费者 成功获取消息" + messageBody);
							StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(messageBody), StuNormScore.class);
							Integer num = mainExecuteStatistic.executeStatistic(stuNormScore);
							if (num == 0) {
								log.info("MQ 消费者处理消息失败");
								return ConsumeConcurrentlyStatus.RECONSUME_LATER;
							}
						}
					} catch (Exception e) {
						log.info("MQ 消费者 处理消息失败");
						log.error(e.getMessage(), e);	
						return ConsumeConcurrentlyStatus.RECONSUME_LATER;
					}finally {
						if (index < msgs.size()) {
                            context.setAckIndex(index + 1);
                        }
					}
					log.info("MQ 消费者 处理消息成功");
					return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
				}
			});
			
			consumer.start();
		} catch (Exception e) {
			log.error("MQ：启动消费者失败：{}-{}");	
			throw new RuntimeException(e.getMessage(), e);
		}
	}
	
	@Override
	public void run(String... arg0) throws Exception {
		this.messageListener();
	}

}

消费者类型2：PullConsumer

package com.becom.qoe.asyncproccess.rocketmq;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.becom.qoe.asyncproccess.entity.StuNormScore;
import com.becom.qoe.asyncproccess.service.MainExecuteStatistic;

import lombok.extern.slf4j.Slf4j;
import net.sf.json.JSONObject;
@Slf4j
@Component
public class RocketMqConsumer{

	@Value("${apache.rocketmq.consumer.ConsumerGroup}")
	private String consumerGroupName;
	@Value("${apache.rocketmq.namesrvAddr}")
	private String namesrvAddr;
	@Value("${apache.rocketmq.producer.topic}")
	private String topic;
	@Value("${apache.rocketmq.producer.tag}")
	private String tag;
	@Autowired
	private MainExecuteStatistic mainExecuteStatistic;
	
	private static final Map<MessageQueue, Long> OFFSE_TABLE = new HashMap<MessageQueue, Long>();
	private DefaultMQPullConsumer pullConsumer = null;
	private static AtomicBoolean RUNNING = new AtomicBoolean(true);
	
	@PostConstruct
	public void messageListener() {
		log.info("MQ：启动消费者");
		pullConsumer = new DefaultMQPullConsumer(consumerGroupName);
		pullConsumer.setNamesrvAddr(namesrvAddr);
		pullConsumer.setVipChannelEnabled(false);
		pullConsumer.setConsumerPullTimeoutMillis(1000 * 10);  //超时时间
		pullConsumer.setBrokerSuspendMaxTimeMillis(1000 * 10);
		pullConsumer.setConsumerTimeoutMillisWhenSuspend(1000 * 15);
		try {
			pullConsumer.start();
			//拉取topic下的所有消息队列
			Set<MessageQueue> mqs = pullConsumer.fetchSubscribeMessageQueues(topic);
			for (MessageQueue mq : mqs) {
				Long endTime = null;
				Long startTime = null;
				while (RUNNING.get()) {
					if (null != endTime && null != startTime) {
						log.info("MQ 消费者 开始拉取消息,与上次拉取时间间隔为：" + (endTime - startTime));
					}
					startTime = System.currentTimeMillis();
					try {
						//设置上次消费消息下标
						PullResult pullResult = pullConsumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 10);
						putMessageQueueOffset(mq, pullResult.getNextBeginOffset());
						switch (pullResult.getPullStatus()) {
							//根据结果状态，如果找到消息，批量消费消息
							case FOUND:
								List<MessageExt> messageExtList = pullResult.getMsgFoundList();
								log.info("MQ 消费者 拉取" + messageExtList.size() + "条消息");
								for (MessageExt m : messageExtList) {
									String messageBody = new String(m.getBody());
									log.info("MQ 消费者 成功获取消息" + messageBody);
									StuNormScore stuNormScore = (StuNormScore) JSONObject.toBean(JSONObject.fromObject(messageBody), StuNormScore.class);
									Integer num = mainExecuteStatistic.executeStatistic(stuNormScore);
									if (num > 0) {
										log.info("MQ 消费者 处理消息成功");
									}
	                            }
								break;
							case NO_MATCHED_MSG:
								log.info("MQ 消费者 没有拉取到匹配的消息");
								break;
							case NO_NEW_MSG:
								endTime = System.currentTimeMillis();
								log.info("MQ 消费者 没有拉取到消息");
								break;
							case OFFSET_ILLEGAL:
								log.info("MQ Consumer offset,may be too big or too small");
								break;
							default:
								break;
						}
					} catch (Exception e) {
						e.printStackTrace();
					} 
				}
			}
		} catch (MQClientException e) {
			e.printStackTrace();
		}
	}
	//获取上次消费的消息的下表
    private static long getMessageQueueOffset(MessageQueue mq) {
        Long offset = (Long) OFFSE_TABLE.get(mq);
        if (offset != null) {
            return offset;
        }
        return 0;
    }
	//保存上次消费的消息下标，这里使用了一个全局HashMap来保存
    private static void putMessageQueueOffset(MessageQueue mq, long offset) {
    	OFFSE_TABLE.put(mq, offset);
    }
	
    @PreDestroy
    public void shutDownConsumer() {
    	RUNNING.set(false);
    	if (pullConsumer != null) {
            pullConsumer.shutdown();
            log.info("MQ Consumer shutDown");
        }
    }
}