1 非docker 安装
https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.11-2.3.0.tgz
tar -zxvf kafka_2.11-2.3.0.tgz
mv kafka_2.11-2.3.0 /usr/local/kafka
vim /usr/local/kafka/config/server.properties
broker.id=1
log.dir=/data/kafka/logs
配置zookeeper 使用kafka/bin/下自带的zk
bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
kafka-zk 需要依赖java环境
apt-get install openjdk-8-jdk
运行 报错 卒。配置低了
bin/kafka-server-start.sh config/server.properties
OpenJDK 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000c0000000, 1073741824, 0) failed; error='Cannot allocate memory' (errno=12)
#
# There is insufficient memory for the Java Runtime Environment to continue.
# Native memory allocation (mmap) failed to map 1073741824 bytes for committing reserved memory.
# An error report file with more information is saved as:
# /usr/local/kafka/hs_err_pid25796.log
2 重新开始
使用docker安装
sudo docker pull wurstmeister/kafka
sudo docker pull wurstmeister/zookeeper
docker-compose.yml
version: "3"
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka:
image: wurstmeister/kafka
depends_on: [ zookeeper ]
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.1.1.1
KAFKA_CREATE_TOPICS: "test:1:1"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- /var/run/docker.sock:/var/run/docker.sock
sudo docker-compose up
报错
Creating kafka_zookeeper_1 ... error
ERROR: for kafka_zookeeper_1 Cannot start service zookeeper: no status provided on response: unknown
ERROR: for zookeeper Cannot start service zookeeper: no status provided on response: unknown
ERROR: Encountered errors while bringing up the project.
换云搬瓦工的机器试一下
docker-compose up -d
Creating network "kafka_default" with the default driver
Creating kafka_zookeeper_1 ... done
Creating kafka_kafka_1 ... done
但是docker ps -a 发现只有zookeeper启动了，kafka失败, 检查日志 发现kafka运行需要java环境，而且对内存有要求，搬瓦工的vps不足够
docker-compose logs -f | grep "kafka"
Attaching to kafka_kafka_1, kafka_zookeeper_1
kafka_1 | waiting for kafka to be ready
kafka_1 | [Configuring] 'advertised.port' in '/opt/kafka/config/server.properties'
kafka_1 | Excluding KAFKA_HOME from broker config
kafka_1 | [Configuring] 'advertised.host.name' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'port' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'broker.id' in '/opt/kafka/config/server.properties'
kafka_1 | Excluding KAFKA_VERSION from broker config
kafka_1 | [Configuring] 'zookeeper.connect' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'log.dirs' in '/opt/kafka/config/server.properties'
kafka_1 | #
kafka_1 | # There is insufficient memory for the Java Runtime Environment to continue.
kafka_1 | # Native memory allocation (mmap) failed to map 1073741824 bytes for committing reserved memory.
kafka_1 | # An error report file with more information is saved as:
kafka_1 | # //hs_err_pid1.log
kafka_1 | OpenJDK 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000c0000000, 1073741824, 0) failed; error='Out of memory' (errno=12)
kafka_1 | waiting for kafka to be ready
kafka_1 | [Configuring] 'advertised.port' in '/opt/kafka/config/server.properties'
kafka_1 | Excluding KAFKA_HOME from broker config
kafka_1 | [Configuring] 'advertised.host.name' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'port' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'broker.id' in '/opt/kafka/config/server.properties'
kafka_1 | Excluding KAFKA_VERSION from broker config
kafka_1 | [Configuring] 'zookeeper.connect' in '/opt/kafka/config/server.properties'
kafka_1 | [Configuring] 'log.dirs' in '/opt/kafka/config/server.properties'
kafka_1 | OpenJDK 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000c0000000, 1073741824, 0) failed; error='Out of memory' (errno=12)
kafka_1 | #
kafka_1 | # There is insufficient memory for the Java Runtime Environment to continue.
kafka_1 | # Native memory allocation (mmap) failed to map 1073741824 bytes for committing reserved memory.
kafka_1 | # An error report file with more information is saved as:
kafka_1 | # /tmp/hs_err_pid1.log
kafka_kafka_1 exited with code 1
^CERROR: Aborting.
因此修改docker-compose.yml 加入以下
KAFKA_HEAP_OPTS: "-Xmx256M -Xms128M"
stop 再启动
docker-compose stop
docker-compose up -d
完美
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
08a4907a4ada wurstmeister/kafka "start-kafka.sh" 22 minutes ago Up 22 minutes 0.0.0.0:9092->9092/tcp kafka_kafka_1
162253966fde wurstmeister/zookeeper "/bin/sh -c '/usr/sb…" About an hour ago Up 22 minutes 22/tcp, 2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp kafka_zookeeper_1
测试
进入容器
docker exec -it kafka_kafka_1 bash
查看已经建好的topic （docker-compose.yml）
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper kafka_zookeeper_1:2181
test
发送消息
$KAFKA_HOME/bin/kafka-console-producer.sh --topic=test --broker-list kafka_kafka_1:9092
>
>xx
>yy
>zz
接收消息
$KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka_kafka_1:9092 --from-beginning --topic test
xx
yy
zz
接下来是golang接入kafka了
package main
import (
"context"
"fmt"
"time"
"github.com/segmentio/kafka-go"
)
var (
addr = "10.1.1.1:9092"
topic = "test"
)
func main() {
go Receive()
go Publish()
time.Sleep(time.Second * 100000)
}
func Receive() {
// make a new reader that consumes from topic-A, partition 0, at offset 42
fmt.Println("receive connecting ...")
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{addr},
Topic: topic,
Partition: 0,
MinBytes: 10e3, // 10KB
MaxBytes: 10e6, // 10MB
//MaxWait: time.Hour,
})
//r.SetOffset(42)
ctx, _ := context.WithTimeout(context.Background(), time.Second*2)
for {
fmt.Println("receive start ...")
m, err := r.ReadMessage(ctx)
fmt.Println("receive one ...")
if err != nil {
fmt.Println("readmessave error", err)
break
}
fmt.Printf("message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value))
}
fmt.Println("receive close")
r.Close()
}
func Publish() {
fmt.Println("publish connecting ...")
w := kafka.NewWriter(kafka.WriterConfig{
Brokers: []string{addr},
Topic: topic,
Balancer: &kafka.LeastBytes{},
})
fmt.Println("publish start ...")
for {
message := "msg" + time.Now().String()
err := w.WriteMessages(context.Background(),
kafka.Message{
Key: []byte("Key-A"),
Value: []byte(message),
},
)
if err != nil {
print("publish err:", err)
return
} else {
fmt.Println("publish sucess")
}
time.Sleep(time.Second)
}
fmt.Println("public close")
w.Close()
}
运行
go run main.go
receive connecting ...
publish connecting ...
publish start ...
receive start ...
message at offset 0: =
message at offset 1: = xx
message at offset 2: = yy
message at offset 3: = zz
message at offset 4: Key-A = msg2019-10-11 17:51:32.305919841 +0800 CST m=+0.000266091
message at offset 5: Key-A = msg2019-10-11 17:53:22.818207089 +0800 CST m=+0.000368552
message at offset 6: Key-A = msg2019-10-11 17:53:26.847210435 +0800 CST m=+4.029372111
message at offset 7: Key-A = msg2019-10-11 17:53:42.670721312 +0800 CST m=+0.000364712