kafkaConsumerClient及kafka多线程重复消费问题

一、kafkaConsumerClient代码

import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.ProducerConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;

@Slf4j
public class KafkaConsumerClient implements ConsumerClient{
    private Boolean isRunning = false;

    private ExecutorService executor;

    private String servers;

    private int sessionTimeout;

    private int maxPollSize;

    private String groupId;

    private String topic;

    private ProcessService processor;

    public KafkaConsumerClient(String servers, int sessionTimeout, int maxPollSize,
                               String groupId, String topic, ProcessService processor) {
        this.servers = servers;
        this.sessionTimeout = sessionTimeout;
        this.maxPollSize = maxPollSize;
        this.groupId = groupId;
        this.topic = topic;
        this.processor = processor;
    }

    public KafkaConsumerClient(String servers, String groupId, String topic, ProcessService processor, String kafkaConsumerQpsMetric) {
        this(servers, 30000, 100, groupId, topic, processor, kafkaConsumerQpsMetric);
    }

    @Override
    public void start(int parallelism) {
        isRunning = true;
        executor = Executors.newFixedThreadPool(parallelism);
        IntStream.range(0, parallelism).forEach(i -> {
            executor.submit(new ConsumerRunner("client" + i));
        });
    }

    @Override
    public void stop() throws InterruptedException {
        isRunning = false;
        executor.shutdown();
        while(!executor.awaitTermination(1, TimeUnit.SECONDS)) {
            log.info("kafka client stopped!");
        }
    }

    class ConsumerRunner implements Runnable{

        private KafkaConsumer consumer;

        private String clientId;

        ConsumerRunner(String clientId){
            Properties props = new Properties();
            props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
            props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
            props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollSize);
            props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
            props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
            props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
            props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
            this.consumer = new KafkaConsumer<>(props);
            this.clientId = clientId;
        }

        @Override
        public void run() {
            try{
                List subscribedTopics = new ArrayList<>();
                //每个 Topic 需要先在控制台进行创建
                subscribedTopics.add(topic);
                consumer.subscribe(subscribedTopics);
                while(isRunning){
                    ConsumerRecords records = consumer.poll(1000);
             		records.forEach(record -> System.out.println(clientId + "-" + record.value()));
                }
                consumer.close();
            }catch(Exception e){
                log.error("kafka consume fail!", e);
            }
        }
    }
}

二、测试过程
1、创建一个topic，里面存放0-99数字。
2、新建client对象，start传入2，两个线程消费，结果发现，两个线程均分了100个数字，并未出现重复消费的情况。
3、传入3，发现各线程分配为33、33、34，基本均匀分配。

client1->7
client1->31
client1->55
client1->79
client1->18
client1->42
client1->66
client1->90
client1->20
client1->44
client1->68
client1->92
client1->5
client1->29
client1->53
client1->77
client1->3
client1->27
client1->51
client1->75
client1->99
client1->17
client1->41
client1->65
client1->89
client1->0
client1->24
client1->48
client1->72
client1->96
client1->8
client1->32
client1->56
client1->80
client1->15
client1->39
client1->63
client1->87
client1->10
client1->34
client1->58
client1->82
client1->22
client1->46
client1->70
client1->94
client1->13
client1->37
client1->61
client1->85
client0->83
client0->21
client0->45
client0->69
client0->93
client0->4
client0->28
client0->52
client0->76
client0->12
client0->36
client0->60
client0->84
client0->19
client0->43
client0->67
client0->91
client0->2
client0->26
client0->50
client0->74
client0->98
client0->9
client0->33
client0->57
client0->81
client0->16
client0->40
client0->64
client0->88
client0->6
client0->30
client0->54
client0->78
client0->14
client0->38
client0->62
client0->86
client0->23
client0->47
client0->71
client0->95
client0->1
client0->25
client0->49
client0->73
client0->97
client0->11
client0->35
client0->59

三、结论

1、consumerConfig可不配置clientId。
2、多线程下kafka会做均匀分配，不会出现重复消费。