截止当前时间,github上golang操作kafka的包主要有两个:
saram 使用纯go语言编写, confluent-kafka-go 这是包装了c的api
Broker
Kafka集群包含一个或多个服务器,这种服务器被称为broker
Topic
每条发布到Kafka集群的消息都有一个类别,这个类别被称为Topic。(物理上不同Topic的消息分开存储,逻辑上一个Topic的消息虽然保存于一个或多个broker上但用户只需指定消息的Topic即可生产或消费数据而不必关心数据存于何处)
Partition
Partition是物理上的概念,每个Topic包含一个或多个Partition.
Producer
负责发布消息到Kafka broker
Consumer
消息消费者,向Kafka broker读取消息的客户端。
Consumer Group
每个Consumer属于一个特定的Consumer Group(可为每个Consumer指定group name,若不指定group name则属于默认的group)。
go version | sarama version |
---|---|
go1.13.5 linux/amd64 | github.com/Shopify/sarama v1.26.1 |
a、 搭建zookeeper集群:
https://blog.csdn.net/luslin1711/article/details/105701958
b、搭建kafka集群
使用docker-compose
version: '3'
services:
broker1:
image: wurstmeister/kafka
restart: always
container_name: broker1
network_mode: host
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9192
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9192
volumes:
- /var/run/docker.sock:/var/run/docker.sock
broker2:
image: wurstmeister/kafka
restart: always
container_name: broker2
network_mode: host
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: 127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9292
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9292
volumes:
- /var/run/docker.sock:/var/run/docker.sock
broker3:
image: wurstmeister/kafka
restart: always
container_name: broker3
network_mode: host
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: 127.0.0.1:2181,127.0.0.1:2181,127.0.0.1:2181,127.0.0.1:2181
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9392
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9392
volumes:
- /var/run/docker.sock:/var/run/docker.sock
kafka-manager:
image: sheepkiller/kafka-manager:latest
network_mode: host
depends_on:
- broker1
- broker2
- broker3
environment:
ZK_HOSTS: 127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184
APPLICATION_SECRET: letmein
KM_ARGS: -Djava.net.preferIPv4Stack=true
producer
package main
import (
"github.com/Shopify/sarama"
"log"
"os"
"os/signal"
"sync"
)
func main() {
config := sarama.NewConfig()
config.Producer.Return.Successes = true
config.Producer.Partitioner = sarama.NewRandomPartitioner
client,err := sarama.NewClient([]string{"localhost:9192","localhost:9292","localhost:9392"}, config)
defer client.Close()
if err != nil {
panic(err)
}
producer, err := sarama.NewAsyncProducerFromClient(client)
if err != nil {
panic(err)
}
// Trap SIGINT to trigger a graceful shutdown.
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
var (
wg sync.WaitGroup
enqueued, successes, errors int
)
wg.Add(1)
// start a groutines to count successes num
go func() {
defer wg.Done()
for range producer.Successes() {
successes++
}
}()
wg.Add(1)
// start a groutines to count error num
go func() {
defer wg.Done()
for err := range producer.Errors() {
log.Println(err)
errors++
}
}()
ProducerLoop:
for {
message := &sarama.ProducerMessage{Topic: "my_topic", Value: sarama.StringEncoder("testing 123")}
select {
case producer.Input() <- message:
enqueued++
case <-signals:
producer.AsyncClose() // Trigger a shutdown of the producer.
break ProducerLoop
}
}
wg.Wait()
log.Printf("Successfully produced: %d; errors: %d\n", successes, errors)
}
consumer
package main
import (
"github.com/Shopify/sarama"
"log"
"os"
"os/signal"
)
func main() {
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
client,err := sarama.NewClient([]string{"localhost:9192","localhost:9292","localhost:9392"}, config)
defer client.Close()
if err != nil {
panic(err)
}
consumer, err := sarama.NewConsumerFromClient(client)
defer consumer.Close()
if err != nil {
panic(err)
}
// get partitionId list
partitions,err := consumer.Partitions("my_topic")
if err != nil {
panic(err)
}
for _, partitionId := range partitions{
// create partitionConsumer for every partitionId
partitionConsumer, err := consumer.ConsumePartition("my_topic", partitionId, sarama.OffsetNewest)
if err != nil {
panic(err)
}
go func(pc *sarama.PartitionConsumer) {
defer (*pc).Close()
// block
for message := range (*pc).Messages(){
value := string(message.Value)
log.Printf("Partitionid: %d; offset:%d, value: %s\n", message.Partition,message.Offset, value)
}
}(&partitionConsumer)
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
select {
case <-signals:
}
}
使用consumer-group进行消费
package main
import (
"context"
"fmt"
"github.com/Shopify/sarama"
"os"
"os/signal"
"sync"
)
type consumerGroupHandler struct{
name string
}
func (consumerGroupHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
func (consumerGroupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
func (h consumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
for msg := range claim.Messages() {
fmt.Printf("%s Message topic:%q partition:%d offset:%d value:%s\n",h.name, msg.Topic, msg.Partition, msg.Offset, string(msg.Value))
// 手动确认消息
sess.MarkMessage(msg, "")
}
return nil
}
func handleErrors(group *sarama.ConsumerGroup,wg *sync.WaitGroup ){
wg.Done()
for err := range (*group).Errors() {
fmt.Println("ERROR", err)
}
}
func consume(group *sarama.ConsumerGroup,wg *sync.WaitGroup, name string) {
fmt.Println(name + "start")
wg.Done()
ctx := context.Background()
for {
topics := []string{"my_topic"}
handler := consumerGroupHandler{name: name}
err := (*group).Consume(ctx, topics, handler)
if err != nil {
panic(err)
}
}
}
func main(){
var wg sync.WaitGroup
config := sarama.NewConfig()
config.Consumer.Return.Errors = false
config.Version = sarama.V0_10_2_0
client,err := sarama.NewClient([]string{"localhost:9192","localhost:9292","localhost:9392"}, config)
defer client.Close()
if err != nil {
panic(err)
}
group1, err := sarama.NewConsumerGroupFromClient("c1", client)
if err != nil {
panic(err)
}
group2, err := sarama.NewConsumerGroupFromClient("c2", client)
if err != nil {
panic(err)
}
group3, err := sarama.NewConsumerGroupFromClient("c3", client)
if err != nil {
panic(err)
}
defer group1.Close()
defer group2.Close()
defer group3.Close()
wg.Add(3)
go consume(&group1,&wg,"c1")
go consume(&group2,&wg,"c2")
go consume(&group3,&wg,"c3")
wg.Wait()
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
select {
case <-signals:
}
}