#kafka 配置 1 spring.kafka.one.bootstrap-servers=%%%% spring.kafka.one.consumer.auto-offset-reset=latest spring.kafka.one.consumer.group-id=prod_maxwell spring.kafka.one.consumer.enable-auto-commit=true spring.kafka.one.listener.concurrency=10 spring.kafka.one.consumer.auto-commit-interval=1000 #kafka 配置2 spring.kafka.two.bootstrap-servers=%%%% spring.kafka.two.consumer.auto-offset-reset=latest spring.kafka.two.consumer.group-id=prod_ib spring.kafka.two.consumer.enable-auto-commit=true spring.kafka.two.consumer.auto-commit-interval=1000 spring.kafka.two.listener.concurrency=12
#Kafka config 1
package com.shandiangou.es.data.kafka.config; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.*; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import java.util.HashMap; import java.util.Map; @EnableKafka @Configuration public class KafkaOneConfig { @Value("${spring.kafka.one.bootstrap-servers}") private String bootstrapServers; @Value("${spring.kafka.one.consumer.group-id}") private String groupId; @Value("${spring.kafka.one.consumer.enable-auto-commit}") private boolean enableAutoCommit; @Value("${spring.kafka.one.consumer.auto-offset-reset}") private String autoOffsetReset; @Value("${spring.kafka.one.listener.concurrency}") private Integer concurrency; @Value("${spring.kafka.one.consumer.auto-commit-interval}") private Integer autoCommitInterval; @Bean public KafkaTemplatekafkaOneTemplate() { return new KafkaTemplate<>(producerFactory()); } @Bean KafkaListenerContainerFactory > kafkaOneContainerFactory() { ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); factory.setConcurrency(concurrency); factory.getContainerProperties().setPollTimeout(3000); return factory; } private ProducerFactory producerFactory() { return new DefaultKafkaProducerFactory<>(producerConfigs()); } public ConsumerFactory consumerFactory() { return new DefaultKafkaConsumerFactory<>(consumerConfigs()); } private Map producerConfigs() { Map props = new HashMap<>(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ProducerConfig.RETRIES_CONFIG, 0); props.put(ProducerConfig.ACKS_CONFIG, "1"); // 不能写成 1 props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); return props; } private Map consumerConfigs() { Map props = new HashMap<>(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,autoOffsetReset); return props; } }
#kafka配置2
package com.shandiangou.es.data.kafka.config; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.config.KafkaListenerContainerFactory; import org.springframework.kafka.core.*; import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; import java.util.HashMap; import java.util.Map; @Configuration public class KafkaTwoConfig { @Value("${spring.kafka.two.bootstrap-servers}") private String bootstrapServers; @Value("${spring.kafka.two.consumer.group-id}") private String groupId; @Value("${spring.kafka.two.consumer.enable-auto-commit}") private boolean enableAutoCommit; @Value("${spring.kafka.two.consumer.auto-offset-reset}") private String autoOffsetReset; @Value("${spring.kafka.two.listener.concurrency}") private Integer concurrency; @Value("${spring.kafka.two.consumer.auto-commit-interval}") private Integer autoCommitInterval; @Bean public KafkaTemplatekafkaTwoTemplate() { return new KafkaTemplate<>(producerFactory()); } @Bean KafkaListenerContainerFactory > kafkaTwoContainerFactory() { ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); factory.setConsumerFactory(consumerFactory()); factory.setConcurrency(concurrency); factory.getContainerProperties().setPollTimeout(3000); return factory; } private ProducerFactory producerFactory() { return new DefaultKafkaProducerFactory<>(producerConfigs()); } public ConsumerFactory consumerFactory() { return new DefaultKafkaConsumerFactory<>(consumerConfigs()); } private Map producerConfigs() { Map props = new HashMap<>(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ProducerConfig.RETRIES_CONFIG, 0); props.put(ProducerConfig.ACKS_CONFIG, "1"); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); return props; } private Map consumerConfigs() { Map props = new HashMap<>(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,autoOffsetReset); return props; } }
#消息监听 这里注意 containerFactory = "kafkaOneContainerFactory" 监听哪个数据源 就写哪个kafka配置的方法
@KafkaListener(topics = "${spring.kafka.topic.itemcore.spu}",containerFactory = "kafkaOneContainerFactory") public void processItemcenterSpuMessage(String spuId) { log.warn("process spuId ={}", spuId); try { long id = Long.valueOf(spuId); spuService.createOrUpdateDocuments(id); } catch (Exception e) { log.error("process spu kafka message fail , message={}", spuId); } }