SpringBoot 整合kafka

参考文章：kafka学习（7）-Springboot整合kafka-多线程

https://spring.io/projects/spring-kafka

springboot集成kafka

pom.xml

  org.springframework.kafka
  spring-kafka

application.yml

spring:
  application:
    name: spring-boot-kafka
  http:
    encoding:
      charset: UTF-8
      force: true
      enabled: true
  kafka:
    client-id: my-client
    #properties:
    #  test: 123
    #admin:
    bootstrap-servers: 10.101.15.229:9092,10.101.15.230:9092,10.101.15.231:9092
    producer:
      #acks
      #batch-size: 16384
      #bootstrap-servers: 10.101.15.163:9092,10.101.15.164:9092,10.101.15.165:9092
      #buffer-memory: 33554432
      #client-id
      #compression-type
      # 指定消息key和消息体的编解码方式
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
    consumer:
      group-id: test
      #auto-offset-reset: earliest
      enable-auto-commit: true
      auto-commit-interval: 1000

      # 指定消息key和消息体的编解码方式
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer

Producer

import io.renren.common.utils.R;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
 
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
 
@RestController
@RequestMapping("test")
public class KafkaTestController {
 
    @Autowired
    private KafkaTemplate kafkaTemplate;
 
    @RequestMapping(value = "/producer")
    public R consume(@RequestBody String body) throws IOException {
        kafkaTemplate.send("result",body);
        return R.ok();
    }
}

Consumer

/**
 * author jinsq
 *
 * @date 2019/5/22 17:06
 */
import io.renren.modules.sys.entity.SysConfigEntity;
import io.renren.modules.sys.service.SysConfigService;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
 
import java.io.IOException;
 
/**
 * kafka监听
 */
@Component
@Slf4j
public class RawDataListener {
    @Autowired
    private SysConfigService sysConfigService;
 
    /**
     * 实时获取kafka数据(生产一条，监听生产topic自动消费一条)
     * @param record
     * @throws IOException
     */
    @KafkaListener(topics = "result")
    public void listen(ConsumerRecord record) throws IOException {
        String value = (String) record.value();
        String topic = record.topic();
        if("result".equals(topic)){
            log.info("接收到的信息为："+value);
            SysConfigEntity sysConfigEntity = new SysConfigEntity();
            sysConfigEntity.setParamKey(value);
            sysConfigEntity.setParamValue(topic);
            sysConfigEntity.setRemark("测试数据");
            sysConfigService.save(sysConfigEntity);
        }else{
            log.info("其他信息！！！！！！！");
        }
    }
 
}

模拟高并发测试

使用CountDownLatch 模拟2000个并发操作

 
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.test.context.junit4.SpringRunner;
 
import java.util.concurrent.CountDownLatch;
 
@RunWith(SpringRunner.class)
@SpringBootTest
public class CountDownLatchTest {
    @Autowired
    private KafkaTemplate kafkaTemplate;
 
    //模拟短时间内的并发请求量
    private static final int threadNum =2000;
    //倒计时器，用于模拟高并发
    private CountDownLatch cdl = new CountDownLatch(threadNum);
    private static int i = 0;
 
    @Test
    public void test(){
        for(int i =1;i<=threadNum;i++){
            MyThread myThread = new MyThread(cdl);
            Thread thread = new Thread(myThread);
            thread.start();
        }
        try {
            cdl.await();
        }catch (Exception e){
            e.printStackTrace();
        }
    }
 
    class MyThread implements Runnable{
        private CountDownLatch countDownLatch;
        public MyThread(CountDownLatch countDownLatch){
            this.countDownLatch = countDownLatch;
        }
        @Override
        public void run(){
            kafkaTemplate.send("result",(i++)+"");
            countDownLatch.countDown();
        }
    }
}

测试结果