go 读写kafka

kafka版： kafka_2.11-2.1.0

1）. 准备
安装依赖库sarama
go get github.com/Shopify/sarama
该库要求kafka版本在0.8及以上，支持kafka定义的high-level API和low-level API，但不支持常用的consumer自动rebalance和offset追踪，所以一般得结合cluster版本使用。
sarama-cluster依赖库
go get github.com/bsm/sarama-cluster
需要kafka 0.9及以上版本
 

package main

import (
   "fmt"
   "github.com/Shopify/sarama"
)

func main() {
   //SaramaProducer()
   //topic := []string{"test"}
   //var wg = &sync.WaitGroup{}
   //wg.Add(1)
   //广播式消费：消费者1
   producer_test("str kafa prof msg")
   go consumer_test()
   //广播式消费：消费者2
   //go clusterConsumer(wg, []string{"127.0.0.1:9092"}, topic, "group-2")

   //wg.Wait()

   select {}

}

var producer sarama.AsyncProducer

func init() {
   fmt.Printf("producer_test\n")
   config := sarama.NewConfig()
   //等待服务器所有副本都保存成功后的响应
   config.Producer.RequiredAcks = sarama.WaitForAll
   //随机向partition发送消息
   config.Producer.Partitioner = sarama.NewRandomPartitioner
   //是否等待成功和失败后的响应,只有上面的RequireAcks设置不是NoReponse这里才有用.
   config.Producer.Return.Successes = true
   config.Producer.Return.Errors = true
   //设置使用的kafka版本,如果低于V0_10_0_0版本,消息中的timestrap没有作用.需要消费和生产同时配置
   //注意，版本设置不对的话，kafka会返回很奇怪的错误，并且无法成功发送消息
   config.Version = sarama.V2_1_0_0

   fmt.Println("build producer")
   //使用配置,新建一个异步生产者
   producer, _ = sarama.NewAsyncProducer([]string{"127.0.0.1:9092"}, config)
   //if e != nil {
   // fmt.Printf("producer_test create producer error :%s\n", e.Error())
   // return
   //}

   //defer producer.AsyncClose()
}

func producer_test(value string) {

   // send message
   msg := &sarama.ProducerMessage{
      Topic: "test",
      //Key:   sarama.StringEncoder("go_test"),
   }

   //value := "this is message"

   //for {
   msg.Value = sarama.ByteEncoder(value)

   // send to chain
   producer.Input() <- msg

   select {
   case <-producer.Successes():
      //fmt.Printf("offset: %d,  timestamp: %s", suc.Offset, suc.Timestamp.String())
   case <-producer.Errors():
      //fmt.Printf("err: %s\n", fail.Err.Error())
   }
   //}
}

func consumer_test() {
   fmt.Printf("consumer_test")

   config := sarama.NewConfig()
   config.Consumer.Return.Errors = true
   config.Version = sarama.V2_1_0_0

   // consumer
   consumer, err := sarama.NewConsumer([]string{"localhost:9092"}, config)
   if err != nil {
      fmt.Printf("consumer_test create consumer error %s\n", err.Error())
      return
   }

   defer consumer.Close()

   partition_consumer, err := consumer.ConsumePartition("test", 0, sarama.OffsetOldest)
   if err != nil {
      fmt.Printf("try create partition_consumer error %s\n", err.Error())
      return
   }
   defer partition_consumer.Close()

   for {
      select {
      case msg := <-partition_consumer.Messages():
         fmt.Printf("msg offset: %d, partition: %d, timestamp: %s, value: %s\n",
            msg.Offset, msg.Partition, msg.Timestamp.String(), string(msg.Value))
      case err := <-partition_consumer.Errors():
         fmt.Printf("err :%s\n", err.Error())
      }
   }
}