kafka与Spring集成

kafka与Spring的集成配置生产者：前提kafka安装完成，及创建好主题

pom文件配置：

    org.apache.kafka
    kafka_2.12
    2.1.0
    
        
            jmxri
            com.sun.jmx
        
        
            jms
            javax.jms
        
        
            jmxtools
            com.sun.jdmk
        
    


    org.apache.kafka
    kafka-clients
    2.1.0


    org.springframework.kafka
    spring-kafka
    2.1.11.RELEASE

config.properties:

#kafka
#kafka访问地址
kafka.serverHost=192.168.3.117:9092
#kafka主题名称
kafka.topic=goods

spring-kafkaProducer.xml：

spring.xml:

KafkaProducerListener监听:

package com.hanshow.wise.base.goods.servlet;

import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.support.ProducerListener;
 
/**
 * kafkaProducer监听器，在producer配置文件中开启
 * @author 
 *
 */
@SuppressWarnings("rawtypes")
public class KafkaProducerListener implements ProducerListener{
    protected final Logger LOG = LoggerFactory.getLogger("kafkaProducer");
    /**
     * 发送消息成功后调用
     */
    public void onSuccess(String topic, Integer partition, Object key,
            Object value, RecordMetadata recordMetadata) {
        LOG.info("==========kafka发送数据成功（日志开始）==========");
        LOG.info("----------topic:"+topic);
        LOG.info("----------partition:"+partition);
        LOG.info("----------key:"+key);
        LOG.info("----------value:"+value);
        LOG.info("----------RecordMetadata:"+recordMetadata);
        LOG.info("~~~~~~~~~~kafka发送数据成功（日志结束）~~~~~~~~~~");
    }
 
    /**
     * 发送消息错误后调用
     */
    public void onError(String topic, Integer partition, Object key,
            Object value, Exception exception) {
        LOG.info("==========kafka发送数据错误（日志开始）==========");
        LOG.info("----------topic:"+topic);
        LOG.info("----------partition:"+partition);
        LOG.info("----------key:"+key);
        LOG.info("----------value:"+value);
        LOG.info("----------Exception:"+exception);
        LOG.info("~~~~~~~~~~kafka发送数据错误（日志结束）~~~~~~~~~~");
        exception.printStackTrace();
    }
 
    /**
     * 方法返回值代表是否启动kafkaProducer监听器
     */
    public boolean isInterestedInSuccess() {
        LOG.info("///kafkaProducer监听器启动///");
        return true;
    }
 
}

KafkaProducerService接口：

package com.hanshow.wise.base.goods.service;

import java.util.Map;

import com.hanshow.wise.base.goods.model.dto.StoreGoodsDTO;
import com.hanshow.wise.common.jo.BaseDTO;

/**
 * kafka接口
 * 
 * @author 
 * @date 2018年6月2日
 * @since 1.0.0
 */
public interface KafkaProducerService {

	BaseDTO> getProducer(StoreGoodsDTO storeGoodsDTO);

}

KafkaProducerServiceImpl实现：

package com.hanshow.wise.base.goods.service.impl;

import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;

import com.hanshow.wise.base.goods.model.dto.StoreGoodsDTO;
import com.hanshow.wise.base.goods.service.KafkaProducerService;
import com.hanshow.wise.base.goods.util.ConfigUtils;
import com.hanshow.wise.common.jo.BaseDTO;

/**
 * 提供商品分类信息查询支持Service实现
 * 
 * @author 
 * @date 2018年6月2日
 * @since 1.0.0
 */
@Service
public class KafkaProducerServiceImpl implements KafkaProducerService {
	
	Logger logger = LoggerFactory.getLogger(KafkaProducerServiceImpl.class);
	
	@Autowired
    private KafkaTemplate kafkaTemplate;

	@Override
	public void getProducer() {
		kafkaTemplate.send(ConfigUtils.getType("kafka.topic"), "内容");
	}
}

测试调用结果：

以上是kafka与Spring的集成内容。

附加简单测试生产者与消费者，无需spring配置。前提kafka安装完成，及创建好主题。

ProducerDemo.java,生产者：

package com.hanshow.wise.base.goods;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;


public class ProducerDemo {
    public static String serverHost="192.168.3.117:9092";
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.servers", serverHost);
        //The "all" setting we have specified will result in blocking on the full commit of the record, the slowest but most durable setting.
        //“所有”设置将导致记录的完整提交阻塞，最慢的，但最持久的设置。
        props.put("acks", "all");
        //如果请求失败，生产者也会自动重试，即使设置成０ the producer can automatically retry.
        props.put("retries", 0);
        //The producer maintains buffers of unsent records for each partition.
        props.put("batch.size", 16384);
        //默认立即发送，这里这是延时毫秒数
        props.put("linger.ms", 1);
        //生产者缓冲大小，当缓冲区耗尽后，额外的发送调用将被阻塞。时间超过max.block.ms将抛出TimeoutException
        props.put("buffer.memory", 33554432);
        //The key.serializer and value.serializer instruct how to turn the key and value objects the user provides with their ProducerRecord into bytes.
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        
        //创建kafka的生产者类
        Producer producer = new KafkaProducer(props);
        for (int i = 0; i < 10; i++) {
            // 这里平均写入４个分区
        	// producer.send(new ProducerRecord("test", i % 4, Integer.toString(i), Integer.toString(i)));
            // 因为没有建立集群所以只能是0个分区
            producer.send(new ProducerRecord("my-first-topic", 0, Integer.toString(i), Integer.toString(i)));
        }
        producer.close();
    }
}

ConsumerDemo.java消费者

package com.hanshow.wise.base.goods;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.util.Arrays;
import java.util.Properties;

public class ConsumerDemo {

    public static void main(String[] args) {
        Properties props = new Properties();

        props.put("bootstrap.servers", ProducerDemo.serverHost);
        System.out.println("this is the group part test 1");
        //消费者的组id
        props.put("group.id", "GroupA");//这里是GroupA或者GroupB
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");

        //从poll(拉)的回话处理时长
        props.put("session.timeout.ms", "30000");
        //poll的数量限制
        //props.put("max.poll.records", "100");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        KafkaConsumer consumer = new KafkaConsumer(props);
        //订阅主题列表topic
        consumer.subscribe(Arrays.asList("my-first-topic"));
        while (true) {
            ConsumerRecords records = consumer.poll(100);
            for (ConsumerRecord record : records)
                //　正常这里应该使用线程池处理，不应该在这里处理
                System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(), record.value() + "\n");

        }
    }

}