KafKa Java编程实例

KafKa Java编程实例

编写一个能发送消息，接收消息的实例

（1）编写系数配置：KafkaProperties.java

package com.imooc.spark.kafka; /** * Kafka常用配置文件 */ public class KafkaProperties {     public static final String ZK = " 140.143.236.169:2181";   //此处要和kafka配置文件中的 advertised.host.name 保持一致，要么都是IP,要么都是HostName     public static final String TOPIC = "hello_topic";     public static final String BROKER_LIST = " 140.143.236.169:9092";     public static final String GROUP_ID = "test_group1"; }

（2）编写生产者： KafkaProducer.java

package com.imooc.spark.kafka; import kafka.javaapi.producer.Producer; import kafka.producer.KeyedMessage; import kafka.producer.ProducerConfig; import java.util.Properties; /** * Kafka生产者 */ public class KafkaProducer extends Thread{     private String topic;     private Producer producer;     public KafkaProducer(String topic) {         this.topic = topic;         Properties properties = new Properties();         properties.put("metadata.broker.list",KafkaProperties.BROKER_LIST);         properties.put("serializer.class","kafka.serializer.StringEncoder");         properties.put("request.required.acks","1");         producer = new Producer(new ProducerConfig(properties));     }     @Override     public void run() {         int messageNo = 1;         while(true) {             String message = "message_" + messageNo;             producer.send(new KeyedMessage(topic, message));             System.out.println("Sent: " + message);             messageNo ++ ;             try{                 Thread.sleep(2000);             } catch (Exception e){                 e.printStackTrace();             }         }     } }

（3）编写消费者：KafkaConsumer.java

package com.imooc.spark.kafka; import kafka.consumer.Consumer; import kafka.consumer.ConsumerConfig; import kafka.consumer.ConsumerIterator; import kafka.consumer.KafkaStream; import kafka.javaapi.consumer.ConsumerConnector; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; /** * Kafka消费者 */ public class KafkaConsumer extends Thread{     private String topic;     public KafkaConsumer(String topic) {         this.topic = topic;     }     private ConsumerConnector createConnector(){         Properties properties = new Properties();         properties.put("zookeeper.connect", KafkaProperties.ZK);         properties.put("group.id",KafkaProperties.GROUP_ID);         return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));     }     @Override     public void run() {         ConsumerConnector consumer = createConnector();         Map topicCountMap = new HashMap();         topicCountMap.put(topic, 1); //        topicCountMap.put(topic2, 1); //        topicCountMap.put(topic3, 1);         // String: topic         // List>  对应的数据流         Map>> messageStream =  consumer.createMessageStreams(topicCountMap);         KafkaStream stream = messageStream.get(topic).get(0);   //获取我们每次接收到的暑假         ConsumerIterator iterator = stream.iterator();         while (iterator.hasNext()) {             String message = new String(iterator.next().message());             System.out.println("rec: " + message);         }     } }

（4）编写启动实例：KafkaClientApp.java

package com.imooc.spark.kafka; /** * Kafka Java API测试 */ public class KafkaClientApp {     public static void main(String[] args) {         new KafkaProducer(KafkaProperties.TOPIC).start();         new KafkaConsumer(KafkaProperties.TOPIC).start();     } }

pom.xml

xsi:schemaLocation=" http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">     4.0.0     com.imooc.spark     sparktrain     1.0     2008              2.11.8         0.9.0.0         2.2.0         2.6.0-cdh5.7.0         1.2.0-cdh5.7.0                                     cloudera              https://repository.cloudera.com/artifactory/cloudera-repos                                         org.scala-lang             scala-library             ${scala.version}                                                          org.apache.hadoop             hadoop-client             ${hadoop.version}                                        org.apache.hbase             hbase-client             ${hbase.version}                               org.apache.hbase             hbase-server             ${hbase.version}                                        org.apache.spark             spark-streaming_2.11             ${spark.version}                                        org.apache.spark             spark-streaming-flume_2.11             ${spark.version}                               org.apache.spark             spark-streaming-flume-sink_2.11             ${spark.version}                               org.apache.spark             spark-streaming-kafka-0-8_2.11             ${spark.version}                                org.apache.commons             commons-lang3             3.5                                        org.apache.spark             spark-sql_2.11             ${spark.version}                               com.fasterxml.jackson.module             jackson-module-scala_2.11             2.6.5                               net.jpountz.lz4             lz4             1.3.0                               mysql             mysql-connector-java             5.1.38                               org.apache.flume.flume-ng-clients             flume-ng-log4jappender             1.6.0                            src/main/scala         src/test/scala                                       org.scala-tools                 maven-scala-plugin                                                                                            compile                             testCompile                                                                                                     ${scala.version}                                              -target:jvm-1.5                                                                                 org.apache.maven.plugins                 maven-eclipse-plugin                                      true                                              ch.epfl.lamp.sdt.core.scalabuilder                                                                   ch.epfl.lamp.sdt.core.scalanature                                                                   org.eclipse.jdt.launching.JRE_CONTAINER                         ch.epfl.lamp.sdt.launching.SCALA_CONTAINER                                                                                                             org.scala-tools                 maven-scala-plugin                                      ${scala.version}