SpringBoot 整合 Kafka

GitHub: link. 欢迎star

注意：本篇博客风格（不多比比就是撸代码!!!）

一、maven依赖

        
        <dependency>
            <groupId>org.springframework.kafkagroupId>
            <artifactId>spring-kafkaartifactId>
            <version>2.7.8version>
        dependency>

二、KafkaProducer.java

import lombok.extern.slf4j.Slf4j;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;

/**
 * @author Andon
 * 2021/11/12
 * 

 * 生产者
 */
@Slf4j
@Component
public class KafkaProducer {
     

    @Resource
    private KafkaTemplate<String, String> kafkaTemplate;

    public void send(String topic, String json) {
     
        kafkaTemplate.send(topic, json);
    }
}

三、KafkaConsumer.java

import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;

import java.util.concurrent.CopyOnWriteArrayList;

/**
 * @author Andon
 * 2021/11/12
 * 

 * 消费者
 */
@Slf4j
@Component
public class KafkaConsumer {
     

    private static CopyOnWriteArrayList<JSONObject> messageList = new CopyOnWriteArrayList<>();

    @KafkaListener(topics = {
     "test01", "test02"}) //监听的topic：test01、test02
    public void consume(ConsumerRecord<String, String> consumerRecord, Acknowledgment acknowledgment) {
     
        if (!ObjectUtils.isEmpty(consumerRecord)) {
     
            String message = consumerRecord.value();
            messageList.add(JSONObject.parseObject(message));
        }
        if (messageList.size() >= 5) {
     
            log.info("messageList:{}", JSONObject.toJSONString(messageList));
            log.info("messageList.size:{}", messageList.size());
            acknowledgment.acknowledge(); //手动提交offset
            messageList.clear();
        }
    }
}

四、application.yml

spring:
  #============== kafka ===================
  kafka:
    # kafka 地址，可以多个，逗号隔开
    bootstrap-servers: ####:9092
    #=============== producer  =======================
    producer:
      # 指定消息key和消息体的编解码方式
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
    #=============== consumer  =======================
    # 指定默认消费者group id
    consumer:
      group-id: test
      # earliest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
      # latest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据
      # none:topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常
      auto-offset-reset: latest
      # 指定消息key和消息体的编解码方式
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      # 关闭自动提交offset
      enable-auto-commit: false
    listener:
      # 消费端监听的topic不存在时，项目启动会报错(关掉)
      missing-topics-fatal: false
      # 手动提交偏移量的ackMode方式
      ack-mode: manual_immediate

五、DatabaseTest.java

import com.alibaba.fastjson.JSONObject;
import com.andon.springbootkafka.kafka.KafkaProducer;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import javax.annotation.Resource;
import java.util.Date;

/**
 * @author Andon
 * 2021/11/12
 */
@RunWith(SpringRunner.class)
@SpringBootTest
public class DatabaseTest {
     

    @Resource
    private KafkaProducer kafkaProducer;

    @Test
    public void test02() {
     
        String topic = "test02";
        String message;
        for (int i = 0; i < 3; i++) {
     
            JSONObject jsonObject = new JSONObject();
            jsonObject.put(String.valueOf(i), new Date().toString());
            message = JSONObject.toJSONString(jsonObject);
            kafkaProducer.send(topic, message);
        }
    }

    @Test
    public void test01() {
     
        String topic = "test01";
        String message;
        for (int i = 0; i < 100; i++) {
     
            message = i + new Date().toString();
            kafkaProducer.send(topic, message);
        }
    }
}

六、测试

GitHub: link. 欢迎star