KAFKA消费者Java实例Demo

 

最近在项目中有个需求，需要接收其他系统的消息--kafka。经过各方借鉴，总结出一个自己比较好理解的Demo，可能还有很多不合理的地方，往见谅，后期会慢慢完善。

package com.picc.ecif.test;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import kafka.consumer.ConsumerConfig;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;

public class KafkaTest {

	private ExecutorService executor;
	private final ConsumerConnector consumer;
	private final String topic;
	
	
	public KafkaTest(String zookeeper, String groupId,String topic) {
		consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig(zookeeper,groupId));
		this.topic = topic;
	}
	
	/**
	 * 消费者参数配置
	 * @param zookeeper zookeeper url地址
	 * @param groupId 消费者组ID
	 * @return
	 */
	private static ConsumerConfig createConsumerConfig(String zookeeper,String groupId){
		Properties props = new Properties();
		props.put("zookeeper.connect", zookeeper);//zookeeper url地址
		props.put("group.id", groupId);//消费者组编号
		props.put("zookeeper.session.timeout.ms", "500");//设置会话响应的时间，超过这个时间kafka可以选择放弃消费或者消费下一条消息
		props.put("zookeeper.sync.time.ms", "250");//leader和follower同步时间
		props.put("auto.commit.interval.ms", "1000");//设置多久一次更新被消费消息的偏移量
		return new ConsumerConfig(props);
	}
	 
	/**
	 * 资源关闭
	 */
	public void shutdown() {
		if (consumer != null)
			consumer.shutdown();
		if (executor != null)
			executor.shutdown();
	}
	
	/**
	 * 同一个topic,多个消费者
	 * @param threadCount
	 */
	public void multiThreadConsumer(int threadCount) {
		//为每个主题定义线程数，多个主题可以添加
		 Map topicMap = new HashMap();
		 topicMap.put(topic, new Integer(threadCount));
		 
		 //根据主题获取消费者信息流
		 Map>> consumerStreamsMap=consumer.createMessageStreams(topicMap);
		 //获取某个主题信息流
		 List> streamList = consumerStreamsMap.get(topic);
		 
		 //启动线程池
		 executor = Executors.newFixedThreadPool(threadCount);
		 
		 //遍历获取到的数据
		 int count = 0;
		 for (final KafkaStream stream : streamList) {
			 final int threadNumber = count;
			 //数据消费部分，该处可以写一个类来实现Runnable接口，来编写数据业务处理逻辑
			 executor.submit(new Runnable() {
				 public void run() {
					 ConsumerIterator consumerIte = stream.iterator();
					 while (consumerIte.hasNext())
						 System.out.println("Thread Number " + threadNumber + ": "+ new String(consumerIte.next().message()));
					 System.out.println("Shutting down Thread Number: " +threadNumber);
				 }
			 });
			 count++;
		 }
		 //消费完数据后关闭资源
		 shutdown();
	 }
	 
	/**
	 * 测试
	 * @param args
	 */
	 public static void main(String[] args) {
		 String zooKeeper = args[0];//zookeeper服务器地址，多个用逗号分隔，如：hostname1:port1,hostname2:port2,hostname3:port3
		 String groupId = args[1];//消费者组ID
		 String topic = args[2];//kafka主题名称
		 int threadCount = Integer.parseInt(args[3]);//线程数
		 KafkaTest multiThreadHLConsumer = new KafkaTest(zooKeeper, groupId, topic);
		 multiThreadHLConsumer.multiThreadConsumer(threadCount);
		 try {
			 Thread.sleep(10000);
		 } catch (InterruptedException ie) {
			 
		 }
		 multiThreadHLConsumer.shutdown();
	 }
}

借鉴于：《Learning Apache Kafka Second Edition》--Nishant Garg