Kafka 接受数据并消费到hbase数据库

一、

   1、生产者 产生数据     

package kafakaTohbase;

import java.util.Properties;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class KafkaProducer {

	public static void main(String[] args) {
		Properties props = new Properties();
		props.put("zk.connect", KafkaProperties.zkConnect);
		props.put("serializer.class", "kafka.serializer.StringEncoder");
		props.put("metadata.broker.list", "hdjt01:9092,hdjt02:9092,hdjt03:9092");
		ProducerConfig config = new ProducerConfig(props);
		Producer producer = new Producer(config);
		for (int i = 0; i < 10; i++){
		  producer.send(new KeyedMessage("test5", "liu" + i));
	}
	}
	

}

　　注： props.put("serializer.class", "kafka.serializer.StringEncoder")  发送的数据是String,

          还可以是 二进制数组形式：        

props.put("serializer.class", "kafka.serializer.DefaultEncoder");
props.put("key.serializer.class", "kafka.serializer.StringEncoder");  如果没有这个，就代表 key也是二进制形式。

         生产者发送的都是keyvalue对

     2、消费者

package kafakaTohbase;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

public class KafkaConsumer extends Thread{

	 private final ConsumerConnector consumer;  
	 private final String topic;  
	  
	    public KafkaConsumer(String topic) {  
	        consumer = kafka.consumer.Consumer  
	                .createJavaConsumerConnector(createConsumerConfig());  
	        this.topic = topic;  
	    }  
	  
	    private static ConsumerConfig createConsumerConfig() {  
	        Properties props = new Properties();  
	        props.put("zookeeper.connect", KafkaProperties.zkConnect);  
	        props.put("group.id", KafkaProperties.groupId1);  
	        props.put("zookeeper.session.timeout.ms", "40000");       //zookeeper 与 region server 的链接超时时间    
	        props.put("zookeeper.sync.time.ms", "200");  
	        props.put("auto.commit.interval.ms", "1000"); 
               //props.put("auto.offset.reset", "smallest");//可以读取旧数据，默认不读取 
	        return new ConsumerConfig(props);  
	    }  
	  
	    @Override  
	    public void run() {  

	        Map topicCountMap = new HashMap();  
	        topicCountMap.put(topic, new Integer(1));  
	        Map>> consumerMap = consumer  
	                .createMessageStreams(topicCountMap);  
	        KafkaStream stream = consumerMap.get(topic).get(0);  
	        ConsumerIterator it = stream.iterator();  
	        HBaseUtils hbase = new HBaseUtils();    
	        while (it.hasNext()) {  //相当于加了一把锁，一直返回true
//	            System.out.println("3receive：" + it.next().message());  
	            try {
	            	System.out.println("11111");
	                hbase.put(new String(it.next().message()));  
	            } catch (IOException e) {  
	                // TODO Auto-generated catch block  
	                e.printStackTrace();  
	            }  
	              
//	          try {  
//	              sleep(300);    // 每条消息延迟300ms  
//	          } catch (InterruptedException e) {  
//	              e.printStackTrace();  
//	          }  
	        }  
	    }  

}

　　连接hbase，配置信息

package kafakaTohbase;

import java.io.IOException;
import java.util.Random;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;

public class HBaseUtils {
	 public  void put(String string) throws IOException {   
		    //设置HBase据库的连接配置参数  
		    Configuration conf = HBaseConfiguration.create();   
		    conf.set("hbase.zookeeper.quorum",  "hdjt01:2181,hdjt02:2181,hdjt03:2181");  //  Zookeeper的地址  
//		    conf.set("hbase.zookeeper.property.clientPort", "42182");   
		    Random random = new Random();  
		    long a = random.nextInt(1000000000);             
		    String tableName = "emp";   
		    String rowkey = "rowkey"+a ;  
		    String columnFamily = "basicinfo";   
		    String column = "empname";   
		    //String value = string;   
		    HTable table=new HTable(conf, tableName);   
		    Put put=new Put(Bytes.toBytes(rowkey));   
		    put.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes.toBytes(string));   
		    table.put(put);//放入表  
		    System.out.println("放入成功");
		    table.close();//释放资源  
		    }  
}

　　测试消费者：

public class Kafkaceshi {

	public static void main(String[] args) {
//		KafkaProducer a=new KafkaProducer ();
//		a.producer();
		KafkaConsumer consumerThread = new KafkaConsumer(KafkaProperties.topic);  
        consumerThread.run();

	}

}

　　

          

转载于:https://www.cnblogs.com/liuwei6/p/6893036.html