大数据基础之Kafka——Kafka Producer和Consumer JAVA API

pom.xml

<dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>2.1.1</version>
    </dependency>

  </dependencies>

Producer类

public class MyProducer {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.56.101:9092");
        prop.put(ProducerConfig.ACKS_CONFIG,"all");
        prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
        prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");

        Producer pro = new KafkaProducer(prop);
        for (int i = 0; i < 10; i++) {
            pro.send(new ProducerRecord<String,String>("test","test"+i,i+""));
        }
        pro.close();
    }
}

Consumer类

public class Consumer {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.56.101:9092");
        prop.put(ConsumerConfig.GROUP_ID_CONFIG,"testgroup1");
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"true");        //自动提交偏移量给zookeeper
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");             //每隔1秒提交一次
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer<String, String> cons = new KafkaConsumer<>(prop);
        cons.subscribe(Arrays.asList("test"));
         while (true){
            ConsumerRecords<String,String> records = cons.poll(Duration.ofSeconds(1));    //每隔一秒消费一次数据
            for (ConsumerRecord<String, String> record : records) {
                System.out.println(record.offset()+":"+record.partition()+":"+record.key()+":"+record.value());
            }
        }
    }
}

多线程分区Consumer

public class MyThreadConsumer extends Thread {
    static Properties prop = new Properties();

    static {
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.56.101:9092");
        prop.put(ConsumerConfig.GROUP_ID_CONFIG, "testgroup1");
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
    }

    private int partitions = 0;

    public MyThreadConsumer(int area) {
        this.partitions = area;
    }

    @Override
    public void run() {
        KafkaConsumer<String, String> cons = new KafkaConsumer<>(prop);
        cons.assign(Arrays.asList(new TopicPartition("mypart", partitions)));
        while (true) {
            ConsumerRecords<String, String> records = cons.poll(Duration.ofSeconds(1));
            for (ConsumerRecord<String, String> record : records) {
                System.out.println("当前线程:"+Thread.currentThread().getName()+" "+record.offset() + ":" + record.partition() + ":" + record.key() + ":" + record.value());
            }
        }
    }


    public static void main(String[] args) {
        for (int i = 0; i < 3; i++) {
            new MyThreadConsumer(i).start();
        }
    }
}