kafka监控获取logSize, offset, lag等信息

由于项目需要，需要查看kafka消费信息lag（lag = logSize - offset）

参考https://www.aliyun.com/jiaocheng/775267.html 的实现方式在有些场景无法获取offset的值（具体原因暂不知晓后续研究下）

因此决定直接从zookeeper中取offset值

一、springboot项目添加依赖

  org.springframework.kafka
  spring-kafka

二、相关代码

import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class KafkaUtil {
	private static Logger logger = LoggerFactory.getLogger(KafkaUtil.class);
	private static final int ZOOKEEPER_TIMEOUT = 30000;
	private final CountDownLatch latch = new CountDownLatch(1);

	public ZooKeeper getZookeeper(String connectionString) {
		ZooKeeper zk = null;
		try {
			zk = new ZooKeeper(connectionString, ZOOKEEPER_TIMEOUT, new Watcher() {
				@Override
				public void process(WatchedEvent event) {
					if (Event.KeeperState.SyncConnected.equals(event.getState())) {
						latch.countDown();
					}
				}
			});
			latch.await();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		return zk;
	}

	public static Properties getConsumerProperties(String groupId, String bootstrap_servers) {
		Properties props = new Properties();
		props.put("group.id", groupId);
		props.put("bootstrap.servers", bootstrap_servers);
		props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
		props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
		return props;
	}

	/**
	 * 获取logSize, offset, lag等信息
	 * @param zk
	 * @param bootstrap_servers
	 * @param groupId
	 * @param topics null查询groupId消费过的所有topic
	 * @param sorted
	 * @return
	 * @throws Exception
	 */
	public List> getLagByGroupAndTopic(ZooKeeper zk, String bootstrap_servers, String groupId,
			String[] topics, boolean sorted) throws Exception {

		List> topicPatitionMapList = new ArrayList<>();

		// 获取group消费过的所有topic
		List topicList = null;
		if (topics == null || topics.length == 0) {
			try {
				topicList = zk.getChildren("/consumers/" + groupId + "/offsets", false);
			} catch (KeeperException | InterruptedException e) {
				logger.error("从zookeeper获取topics失败：zkState: {}, groupId:{}", zk.getState(), groupId);
				throw new Exception("从zookeeper中获取topics失败");
			}
		} else {
			topicList = Arrays.asList(topics);
		}

		Properties consumeProps = getConsumerProperties(groupId, bootstrap_servers);
		logger.info("consumer properties:{}", consumeProps);
		KafkaConsumer consumer = new KafkaConsumer<>(consumeProps);

		// 查询topic partitions
		for (String topic : topicList) {
			List partitionsFor = consumer.partitionsFor(topic);
			//由于有时延， 尽量逐个topic查询， 减少lag为负数的情况
			List topicPartitions = new ArrayList<>();

			// 获取topic对应的 TopicPartition
			for (PartitionInfo partitionInfo : partitionsFor) {
				TopicPartition topicPartition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
				topicPartitions.add(topicPartition);
			}
			// 查询logSize
			Map endOffsets = consumer.endOffsets(topicPartitions);
			for (Entry entry : endOffsets.entrySet()) {
				TopicPartition partitionInfo = entry.getKey();
				// 获取offset
				String offsetPath = MessageFormat.format("/consumers/{0}/offsets/{1}/{2}", groupId, partitionInfo.topic(),
						partitionInfo.partition());
				byte[] data = zk.getData(offsetPath, false, null);
				long offset = Long.valueOf(new String(data));

				Map topicPatitionMap = new HashMap<>();
				topicPatitionMap.put("group", groupId);
				topicPatitionMap.put("topic", partitionInfo.topic());
				topicPatitionMap.put("partition", partitionInfo.partition());
				topicPatitionMap.put("logSize", endOffsets.get(partitionInfo));
				topicPatitionMap.put("offset", offset);
				topicPatitionMap.put("lag", endOffsets.get(partitionInfo) - offset);
				topicPatitionMapList.add(topicPatitionMap);
			}
		}
		consumer.close();
		
		if(sorted) {
			Collections.sort(topicPatitionMapList, new Comparator>() {
				@Override
				public int compare(Map o1, Map o2) {
					if(o1.get("topic").equals(o2.get("topic"))) {
						return ((Integer)o1.get("partition")).compareTo((Integer)o2.get("partition"));
					}
					return ((String)o1.get("topic")).compareTo((String)o2.get("topic"));
				}
			});
		}
		
		return topicPatitionMapList;
	}

	public static void main(String[] args) throws Exception {
		String bootstrap_servers = "localhost:9092";
		String groupId = "interface-group-new";
		String[] topics = null;//{"test1", "test2", test3};

		KafkaUtil kafkaUtil = new KafkaUtil();
		String connectionString = "localhost:2181";
		ZooKeeper zk = kafkaUtil.getZookeeper(connectionString);
		if (zk == null) {
			throw new RuntimeException("获取zookeeper连接失败");
		}
		List> topicPatitionMapList = kafkaUtil.getLagByGroupAndTopic(zk, bootstrap_servers,
				groupId, topics, true);

		for (Map map : topicPatitionMapList) {
			System.out.println(map);
		}
		zk.close();
	}
}

三、说明

 调用时参数topics为空会获取到groupId所有消费过的topic（zookeeper会保存消费过的groupId的offset值）

List partitionsFor = consumer.partitionsFor(topic);

  获取到 List 后要尽快查询zookeeper对应的offset，避免由于继续生产消费或时延导致offset > logSize

参考：https://www.aliyun.com/jiaocheng/775267.html