kafka producer 发送消息

package com.zhp.springbootstreamdemo.kafka;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;

public class ProducerDemo {
    public static void main(String[] args) {
        //消息发送模式：同步或异步
        boolean isAsync = args.length == 0
                || !args[0].trim().equalsIgnoreCase("sync");

        Properties properties = new Properties();
        //Kafka服务端的主机名和端口号
        properties.put("bootstrap.servers", "localhost:9092");
        //客户的ID
        properties.put("client.id", "ProducerDemo");
        //消息的key和value都是字节数组，为了将Java对象转化为字节数组，可以配置
        //key.serializer和value.serializer两个序列化器，完成转化
        properties.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
        // StringSerializer用来将String对象序列化成字节数组
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        //生产者核心类
        KafkaProducer<Integer, String> producer = new KafkaProducer<>(properties);
        String topic = "test";
        //消息的Key
        int messageKey = 1;
        while (true) {
            // 消息的value
            String messageValue = "Message_" + messageKey;
            long startTime = System.currentTimeMillis();
            if (isAsync) {
                //异步发送消息
                // 第一个参数是ProducerRecord类型的对象，封装了目标Topic、消息的key、消息的value
                // 第二个参数是一个CallBack对象，当生产者接收到Kafka发来的ACK确
                // 认消息的时候，会调用此CallBack对象的onCompletion()方法，实现 回调功能
                ProducerRecord<Integer, String> record = new ProducerRecord<>(topic, messageKey, messageValue);
                producer.send(record, new DemoCallBack<>(startTime, messageKey, messageValue));
            } else {
                //同步发送消息
                //KafkaProducer.send()方法的返回值类型是Future
                //这里通过Future.get()方法，阻塞当前线程，等待Kafka服务端的ACK响应
                ProducerRecord<Integer, String> producerRecord = new ProducerRecord<>(topic, messageKey, messageValue);
                try {
                    RecordMetadata recordMetadata = producer.send(producerRecord).get();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
            messageKey++;
        }
    }
}

class DemoCallBack<K, V> implements Callback {
    private final long startTime;
    private final K key;
    private final V value;

    public DemoCallBack(long startTime, K key, V value) {
        this.startTime = startTime;
        this.key = key;
        this.value = value;
    }

    /**
     * 生产者成功发送消息，收到Kafka服务端发来的ACK确认消息后，会调用此回调函数
     *
     * @param recordMetadata 生产者发送的消息的元数据，如果发送过程中出现异常，此参数为null
     * @param exception      发送过程中出现的异常，如果发送成功，则此参数为null
     */
    @Override
    public void onCompletion(RecordMetadata recordMetadata, Exception exception) {
        if (recordMetadata != null) {
            long elapsedTime = System.currentTimeMillis() - startTime;
            System.out.println("message(" + key + "," + value + ") send to partition("
                    + recordMetadata.partition() + ")," + "offset(" + recordMetadata.offset() + ") in" + elapsedTime);
        } else {
            exception.printStackTrace();
        }
    }

}

发送的消息如何选择分区：

在KafkaProducer的doSend方法中有如下代码：

int partition = partition(record, serializedKey, serializedValue, cluster);

private int partition(ProducerRecord<K, V> record, byte[] serializedKey, byte[] serializedValue, Cluster cluster) {
    Integer partition = record.partition();
    return partition != null ?
            partition :
            partitioner.partition(
                    record.topic(), record.key(), serializedKey, record.value(), serializedValue, cluster);
}

如果我们创建 ProducerRecord时设置了 partition，就用我们设置的。否则进行计算。

默认的实现是DefaultPartitioner

public static final String PARTITIONER_CLASS_CONFIG = "partitioner.class";

.define(PARTITIONER_CLASS_CONFIG,
        Type.CLASS,
        DefaultPartitioner.class,
        Importance.MEDIUM, PARTITIONER_CLASS_DOC)

如果我们没有配置就使用默认的。

我们看默认实现方法

public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    if (keyBytes == null) {
        int nextValue = nextValue(topic);
        List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic);
        if (availablePartitions.size() > 0) {
            int part = Utils.toPositive(nextValue) % availablePartitions.size();
            return availablePartitions.get(part).partition();
        } else {
            // no partitions are available, give a non-available partition
            return Utils.toPositive(nextValue) % numPartitions;
        }
    } else {
        // hash the keyBytes to choose a partition
        return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
    }
}

private int nextValue(String topic) {
    AtomicInteger counter = topicCounterMap.get(topic);
    if (null == counter) {
        counter = new AtomicInteger(ThreadLocalRandom.current().nextInt());
        AtomicInteger currentCounter = topicCounterMap.putIfAbsent(topic, counter);
        if (currentCounter != null) {
            counter = currentCounter;
        }
    }
    return counter.getAndIncrement();
}

如果keyBytes为null 就使用轮询选择分区。否则根据key的hash值。