Java连接Kafka实例

目录

1 Java连接

1.1 Maven依赖

1.2 生产者代码

1.3 消费者代码

2 Spring Boot连接

2.1 Maven依赖

2.2 配置文件

2.3 生产者代码

2.4 消费者代码

3 Spring Cloud Stream连接

3.1 Maven依赖

3.2 配置文件

3.3 自定义Channel

3.4 生产者代码

3.5 消费者代码

---

笔者使用的Kafka为Windows版本（Kafka 2.3.1），所以在下面的连接中使用的是本地连接。如果使用的是虚拟机远程连接的话，ip地址只需修改成相应虚拟机的ip地址即可。

---

1 Java连接

1.1 Maven依赖

    org.apache.kafka
    kafka-clients
    2.3.1



    org.slf4j
    slf4j-api
    1.7.28

1.2 生产者代码

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;

public class ProducerFastStart {

    public static final String BROKER_LIST = "localhost:9092";
    public static final String TOPIC = "topic-demo";

    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BROKER_LIST);
        //构建所需要发送的消息
        ProducerRecord record = new ProducerRecord<>(TOPIC, "Hello Kafka!");
        //配置生产者客户端参数并创建KafkaProducer示例
        try (KafkaProducer producer = new KafkaProducer<>(properties)) {
            //发送消息
            producer.send(record, (metadata, exception) -> {
                if (exception != null) {
                    exception.printStackTrace();
                } else {
                    System.out.println(metadata.topic() + "-" + metadata.partition() + ":" + metadata.offset());
                }
            });
        }
    }
}

1.3 消费者代码

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;

public class KafkaConsumerAnalysis {

    public static final String BROKER_LIST = "localhost:9092";
    public static final String TOPIC = "topic-demo";
    public static final String GROUP_ID = "group.demo";
    public static final AtomicBoolean isRunning = new AtomicBoolean(true);

    private static Properties initConfig() {
        Properties props = new Properties();
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BROKER_LIST);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID);
        props.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer.client.id.demo");
        return props;
    }

    public static void main(String[] args) {
        Properties props = initConfig();
        KafkaConsumer consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(TOPIC));
        try {
            while (isRunning.get()) {
                ConsumerRecords records = consumer.poll(Duration.ofMillis(1000));
                for (ConsumerRecord record : records) {
                    System.out.println("topic=" + record.topic() + ", partition=" + record.partition() + ", offset=" + record.offset());
                    System.out.println("key=" + record.key() + ", value=" + record.value());
                }
            }
        } finally {
            consumer.close();
        }
    }
}

---

2 Spring Boot连接

2.1 Maven依赖

    4.0.0
    
        org.springframework.boot
        spring-boot-starter-parent
        2.2.4.RELEASE
         
    
    com.hys
    kafka
    0.0.1-SNAPSHOT
    kafka
    Demo project for Spring Boot

    
        1.8
    

    
        
            org.springframework.boot
            spring-boot-starter-web
        
        
            org.springframework.boot
            spring-boot-starter
        
        
            org.springframework.kafka
            spring-kafka
        

        
            org.springframework.boot
            spring-boot-starter-test
            test
            
                
                    org.junit.vintage
                    junit-vintage-engine
                
            
        
        
            org.springframework.kafka
            spring-kafka-test
            test
        
    

    
        
            
                org.springframework.boot
                spring-boot-maven-plugin
            
        
    

2.2 配置文件

#生产者
spring.kafka.producer.bootstrap-servers=localhost:9092
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
#消费者
spring.kafka.consumer.bootstrap-servers=localhost:9092
spring.kafka.consumer.group-id=foo
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

2.3 生产者代码

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;

@RestController
public class SimpleController {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    @GetMapping("/send/{message}")
    public void send(@PathVariable String message) {
        kafkaTemplate.send("mytopic", message);
    }
}

2.4 消费者代码

import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

@Component
public class SimpleListener {

    @KafkaListener(topics = {"mytopic"})
    public void listen(String data) {
        System.out.println(data);
    }
}

---

3 Spring Cloud Stream连接

3.1 Maven依赖

    4.0.0
    
        org.springframework.boot
        spring-boot-starter-parent
        2.2.4.RELEASE
         
    
    com.hys
    spring-cloud-stream-kafka
    0.0.1-SNAPSHOT
    spring-cloud-stream-kafka
    Demo project for Spring Boot

    
        1.8
        Hoxton.SR1
    

    
        
            org.springframework.boot
            spring-boot-starter-web
        
        
            org.apache.kafka
            kafka-streams
        
        
            org.springframework.cloud
            spring-cloud-starter
        
        
            org.springframework.kafka
            spring-kafka
        
        
            org.springframework.cloud
            spring-cloud-stream-binder-kafka
        

        
            org.springframework.boot
            spring-boot-starter-test
            test
            
                
                    org.junit.vintage
                    junit-vintage-engine
                
            
        
        
            org.springframework.kafka
            spring-kafka-test
            test
        
    

    
        
            
                org.springframework.cloud
                spring-cloud-dependencies
                ${spring-cloud.version}
                pom
                import
            
        
    

    
        
            
                org.springframework.boot
                spring-boot-maven-plugin
            
        
    

3.2 配置文件

server.port=8080
spring.application.name=kafka-stream
spring.cloud.stream.kafka.binder.brokers=localhost:9092
spring.cloud.stream.bindings.mychannel-input.destination=mytopic
spring.cloud.stream.bindings.mychannel-output.destination=mytopic
spring.cloud.stream.bindings.mychannel-output.content-type=text/plain
#消息分组
spring.cloud.stream.bindings.mychannel-input.group=g1
spring.cloud.stream.bindings.mychannel-output.group=g1

3.3 自定义Channel

import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.SubscribableChannel;

public interface MyChannel {

    String INPUT = "mychannel-input";
    String OUTPUT = "mychannel-output";

    @Output(OUTPUT)
    MessageChannel output();

    @Input(INPUT)
    SubscribableChannel input();
}

3.4 生产者代码

import com.hys.kafka.service.MyChannel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
public class HelloController {

    @Autowired
    private MyChannel myChannel;

    @GetMapping("test")
    public void hello() {
        myChannel.output().send(MessageBuilder.withPayload("hello stream!").build());
    }
}

3.5 消费者代码

import com.hys.kafka.service.MyChannel;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;

@EnableBinding(MyChannel.class)
public class MsgReceiver {

    @StreamListener(MyChannel.INPUT)
    public void receive(Object payload) {
        System.out.println("Received:" + payload);
    }
}