kafka 两台单机实现 数据同步

kafka数据同步

zookeeper配置

[root@node1 zookeeper-3.4.9]# grep '^[a-Z]' conf/zoo.cfg 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/iotplatform/zookeeper-3.4.9/data
dataLogDir=/opt/iotplatform/zookeeper-3.4.9/datalog
clientPort=2181

kafka配置

# 修改配置文件如下
[root@node1 kafka_2.11-2.3.0]# grep '^[a-Z]' config/server.properties
broker.id=1
listeners=PLAINTEXT://node1.iotplatform.com:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/iotplatform/kafka_2.11-2.3.0/datalog
num.partitions=5
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=24
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=node1.iotplatform.com:2181
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=0


#node1.iotplatform.com:添加以下配置
[root@node1 kafka_2.11-2.3.0]# cat sourceClusterConsumer.config 
zookeeper.connect=node1.iotplatform.com:2181
bootstrap.servers=node1.iotplatform.com:9092
group.id=1
partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor

[root@node1 kafka_2.11-2.3.0]# cat targetClusterProducer.config 
zookeeper.connect=node2.iotplatform.com:2181
bootstrap.servers=node2.iotplatform.com:9092


nohup bin/kafka-mirror-maker.sh  --consumer.config sourceClusterConsumer.config --num.streams 2 --producer.config targetClusterProducer.config -whitelist=".*" &

如果node1节点挂掉之后,启用node2

# node2.iotplatform.com:添加以下配置
[root@node1 kafka_2.11-2.3.0]# cat sourceClusterConsumer.config 
zookeeper.connect=node2.iotplatform.com:2181
bootstrap.servers=node2.iotplatform.com:9092
group.id=1
partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor

[root@node1 kafka_2.11-2.3.0]# cat targetClusterProducer.config 
zookeeper.connect=node1.iotplatform.com:2181
bootstrap.servers=node1.iotplatform.com:9092

# 启动mirror-maker
nohup bin/kafka-mirror-maker.sh  --consumer.config sourceClusterConsumer.config --num.streams 2 --producer.config targetClusterProducer.config -whitelist=".*" &