kafka+connector集群迁移

示例环境

ip	部署信息	备注
192.168.181.121	zookeeper、kafka、connecter	待迁移集群节点1
192.168.181.14	zookeeper、kafka、connecter	待迁移集群节点2
192.168.181.49	zookeeper、kafka、connecter	待迁移集群节点3
192.168.181.61	zookeeper、kafka、connecter	新集群节点1
192.168.181.66	zookeeper、kafka、connecter	新集群节点2
192.168.181.70	zookeeper、kafka、connecter	新集群节点3

一、运行测试脚本，分别生产消息和消费消息

producer.py

from kafka import KafkaProducer
import json
import time
 
def create_producer():
    producer = KafkaProducer(
        bootstrap_servers='192.168.181.121:9092,192.168.181.14:9092,192.168.181.49:9092',  # Kafka 服务器地址
        value_serializer=lambda v: json.dumps(v).encode('utf-8')  # 序列化为 JSON
    )
    return producer
 
def send_messages(producer, topic):
    for i in range(100000):
        message = {'number': i}
        producer.send(topic, value=message)
        print(f'Sent: {message}')
        time.sleep(2)  # 每秒发送一条消息
 
if __name__ == '__main__':
    producer = create_producer()
    send_messages(producer, 'testa')  # 替换为你想要的主题
    producer.flush()  # 确保所有消息都被发送
    producer.close()

consumer.py

from kafka import KafkaConsumer
import json
 
def create_consumer():
    consumer = KafkaConsumer(
        'testa',  # 替换为你想要的主题
        bootstrap_servers='192.168.181.121:9092,192.168.181.14:9092,192.168.181.49:9092',  # Kafka 服务器地址
        auto_offset_reset='earliest',  # 从最早的消息开始消费
        value_deserializer=lambda x: json.loads(x.decode('utf-8')),  # 反序列化为 JSON
        group_id='my-group'  # 消费者组
    )
    return consumer
 
def consume_messages(consumer):
    for message in consumer:
        print(f'Received: {message.value}')
 
if __name__ == '__main__':
    consumer = create_consumer()
    consume_messages(consumer)

python producer.py
 
Sent: {'number': 0}
Sent: {'number': 1}
Sent: {'number': 2}
......
 
python consumer.py
 
Received: {'number': 0}
Received: {'number': 1}
Received: {'number': 2}
......

二、添加新zookeeper节点到集群中

修改zookeeper配置文件

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# the directory where the snapshot is stored.
dataDir=/data/kafka/data/zookeeper
# the port at which the clients will connect
clientPort=2181
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
tickTime=2000
initLimit=10
syncLimit=5
 
server.1=192.168.181.121:2888:3888
server.2=192.168.181.14:2888:3888
server.3=192.168.181.49:2888:3888
server.4=192.168.181.61:2888:3888
server.5=192.168.181.66:2888:3888
server.6=192.168.181.70:2888:3888

新节点创建myid

#192.168.181.61
[root@181-61 kafka]# echo "4" > data/zookeeper/myid
 
#192.168.181.66
[root@181-66 kafka]# echo "5" > data/zookeeper/myid
 
#192.168.181.70
[root@181-70 kafka]# echo "6" > data/zookeeper/myid

启动新节点上的zookeeper

#192.168.181.61
[root@181-61 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.66
[root@181-66 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.70
[root@181-70 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

重启旧集群zookeeper

#192.168.181.61
[root@181-61 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.66
[root@181-66 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.70
[root@181-70 kafka]# ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

在leader上查询集群信息
zk_followers 5
zk_synced_followers 5

[root@181-49 kafka]# echo mntr|nc localhost 2181
zk_version      3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf, built on 03/06/2019 16:18 GMT
zk_avg_latency  0
zk_max_latency  2
zk_min_latency  0
zk_packets_received     117
zk_packets_sent 116
zk_num_alive_connections        4
zk_outstanding_requests 0
zk_server_state leader
zk_znode_count  221
zk_watch_count  24
zk_ephemerals_count     4
zk_approximate_data_size        18069
zk_open_file_descriptor_count   127
zk_max_file_descriptor_count    65535
zk_fsync_threshold_exceed_count 0
zk_followers    5
zk_synced_followers     5
zk_pending_syncs        0
zk_last_proposal_size   -1
zk_max_proposal_size    -1
zk_min_proposal_size    -1

三、添加新kafka节点到集群中

修改server.properties, zookeeper地址填写三个新zookeeper节点地址

broker.id=3
listeners=PLAINTEXT://192.168.181.61:9092
advertised.listeners=PLAINTEXT://192.168.181.61:9092
num.network.threads=3
 
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
 
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
default.replication.factor=3
auto.create.topics.enable=true
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.181.66:2181,192.168.181.70:2181,192.168.181.61:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0

启动kafka服务

#192.168.181.61
[root@181-61 kafka]# ./bin/kafka-server-start.sh -daemon config/server.properties
 
#192.168.181.66
[root@181-66 kafka]# ./bin/kafka-server-start.sh -daemon config/server.properties
 
#192.168.181.70
[root@181-70 kafka]# ./bin/kafka-server-start.sh -daemon config/server.properties

查看kafka集群信息，节点添加成功

四、迁移kafka数据到新节点（3,4,5）中

1、确认要移动的topics

[root@181-121 kafka]# ./bin/kafka-topics.sh --list --zookeeper 127.0.0.1:2181
__consumer_offsets
cdc_md_combine
cdc_mdata_sdp4_PR04144_XJGX_1
cdc_mdata_sdp4_PR04144_XJGX_a1
connect-configs
connect-offsets
connect-status
testa

2、复制这些topic,并写成如下格式的文件, 命名为 topics-to-move.json

{"topics": [
 {"topic": "__consumer_offsets"},
 {"topic": "cdc_md_combine"},
 {"topic": "cdc_mdata_sdp4_PR04144_XJGX_1"},
 {"topic": "cdc_mdata_sdp4_PR04144_XJGX_a1"},
 {"topic": "connect-configs"},
 {"topic": "connect-offsets"},
 {"topic": "connect-status"},
 {"topic": "testa"}
 ],
 "version":1
}

3、生成移动脚本

其中3,4,5是新kafka 节点的broker.id

将Proposed partition reassignment configuration下面的内容写入reassignment-node.json中

[root@181-121 kafka]# bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --topics-to-move-json-file  topics-to-move.json --broker-list "3,4,5" --generate
Current partition replica assignment
{"version":1,"partitions":[{"topic":"__consumer_offsets","partition":22,"replicas":[2],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_1","partition":0,"replicas":[0,2,1],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":30,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":20,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":8,"replicas":[0],"log_dirs":["any"]},{"topic":"testa","partition":0,"replicas":[2,0,1],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":21,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":27,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":7,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":9,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":46,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":7,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":24,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":16,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":25,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-status","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":35,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":41,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":33,"replicas":[1],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_a1","partition":0,"replicas":[1,2,0],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":23,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":49,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":3,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":21,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":11,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":47,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":6,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":16,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-configs","partition":0,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":28,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":31,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":36,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-status","partition":3,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":42,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":3,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":18,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":17,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":37,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":10,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":15,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":24,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":2,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":18,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":23,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-status","partition":2,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":38,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":17,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":48,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":19,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":11,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":15,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":13,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":2,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":43,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":6,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":14,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":12,"replicas":[0],"log_dirs":["any"]},{"topic":"cdc_md_combine","partition":0,"replicas":[0,1,2],"log_dirs":["any","any","any"]},{"topic":"connect-offsets","partition":5,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":13,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":14,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-status","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":20,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":0,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":44,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":8,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":39,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":12,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":45,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":5,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":26,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":9,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":29,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-status","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":19,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":22,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":34,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":10,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":32,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":40,"replicas":[2],"log_dirs":["any"]}]}
 
Proposed partition reassignment configuration
{"version":1,"partitions":[{"topic":"__consumer_offsets","partition":49,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":38,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":10,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":16,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":27,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":19,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":8,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":4,"replicas":[5],"log_dirs":["any"]},{"topic":"testa","partition":0,"replicas":[4,3,5],"log_dirs":["any","any","any"]},{"topic":"cdc_md_combine","partition":0,"replicas":[4,5,3],"log_dirs":["any","any","any"]},{"topic":"connect-offsets","partition":15,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-status","partition":2,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":2,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":13,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":24,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":46,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":35,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":12,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":1,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":5,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":23,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":43,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":32,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":21,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":10,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":37,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":20,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":9,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-status","partition":4,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":48,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":40,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":29,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":18,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":14,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":7,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":34,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":45,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":23,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":6,"replicas":[4],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_a1","partition":0,"replicas":[5,3,4],"log_dirs":["any","any","any"]},{"topic":"connect-offsets","partition":17,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-status","partition":1,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":0,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":22,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":26,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":11,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":15,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":4,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":42,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":20,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":31,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":3,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":9,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":1,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":12,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-status","partition":3,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":8,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":19,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":17,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":28,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":6,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":39,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":44,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":16,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-status","partition":0,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":5,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":36,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":21,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":47,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":14,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":3,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":25,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":30,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":41,"replicas":[5],"log_dirs":["any"]},{"topic":"connect-offsets","partition":2,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":24,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":13,"replicas":[5],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_1","partition":0,"replicas":[4,5,3],"log_dirs":["any","any","any"]},{"topic":"connect-configs","partition":0,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":11,"replicas":[5],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":22,"replicas":[4],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":33,"replicas":[3],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":0,"replicas":[3],"log_dirs":["any"]},{"topic":"connect-offsets","partition":18,"replicas":[4],"log_dirs":["any"]},{"topic":"connect-offsets","partition":7,"replicas":[5],"log_dirs":["any"]}]}

4、开始进行数据迁移

[root@181-121 kafka]# bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --reassignment-json-file  reassignment-node.json --execute
Current partition replica assignment
 
{"version":1,"partitions":[{"topic":"__consumer_offsets","partition":22,"replicas":[2],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_1","partition":0,"replicas":[0,2,1],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":30,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":20,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":8,"replicas":[0],"log_dirs":["any"]},{"topic":"testa","partition":0,"replicas":[2,0,1],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":21,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":27,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":7,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":9,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":46,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":7,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":24,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":16,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":25,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-status","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":35,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":41,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":33,"replicas":[1],"log_dirs":["any"]},{"topic":"cdc_mdata_sdp4_PR04144_XJGX_a1","partition":0,"replicas":[1,2,0],"log_dirs":["any","any","any"]},{"topic":"__consumer_offsets","partition":23,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":49,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":3,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":21,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":11,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":47,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":6,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":16,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-configs","partition":0,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":28,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":31,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":36,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-status","partition":3,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":42,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":3,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":18,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":17,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":37,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":10,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":15,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":24,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":2,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":18,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":23,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-status","partition":2,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":38,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":17,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":48,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":19,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":11,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":15,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":13,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":2,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":4,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":43,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":6,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":14,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":12,"replicas":[0],"log_dirs":["any"]},{"topic":"cdc_md_combine","partition":0,"replicas":[0,1,2],"log_dirs":["any","any","any"]},{"topic":"connect-offsets","partition":5,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":13,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":14,"replicas":[1],"log_dirs":["any"]},{"topic":"connect-offsets","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-status","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":20,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":0,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":44,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":8,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":39,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":12,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":45,"replicas":[1],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":5,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":26,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-offsets","partition":9,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":29,"replicas":[0],"log_dirs":["any"]},{"topic":"connect-status","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":1,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":19,"replicas":[2],"log_dirs":["any"]},{"topic":"connect-offsets","partition":22,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":34,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":10,"replicas":[2],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":32,"replicas":[0],"log_dirs":["any"]},{"topic":"__consumer_offsets","partition":40,"replicas":[2],"log_dirs":["any"]}]}
 
Save this to use as the --reassignment-json-file option during rollback
Successfully started reassignment of partitions.

查询运行结果,数据已迁移至新节点中

[root@181-121 kafka]# bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --reassignment-json-file reassignment-node.json --verify
Status of partition reassignment:
Reassignment of partition __consumer_offsets-22 completed successfully
Reassignment of partition cdc_mdata_sdp4_PR04144_XJGX_1-0 completed successfully
Reassignment of partition __consumer_offsets-30 completed successfully
Reassignment of partition connect-offsets-20 completed successfully
Reassignment of partition __consumer_offsets-8 completed successfully
Reassignment of partition testa-0 completed successfully
Reassignment of partition __consumer_offsets-21 completed successfully
Reassignment of partition __consumer_offsets-4 completed successfully
Reassignment of partition __consumer_offsets-27 completed successfully
Reassignment of partition __consumer_offsets-7 completed successfully
Reassignment of partition __consumer_offsets-9 completed successfully
Reassignment of partition __consumer_offsets-46 completed successfully
Reassignment of partition connect-offsets-7 completed successfully
Reassignment of partition connect-offsets-24 completed successfully
Reassignment of partition connect-offsets-16 completed successfully
Reassignment of partition __consumer_offsets-25 completed successfully
Reassignment of partition connect-status-4 completed successfully
......

5、重新项目服务，使用kafka新broker节点进行连接

def create_producer():
    producer = KafkaProducer(
        bootstrap_servers='192.168.181.61:9092,192.168.181.66:9092,192.168.181.70:9092',  # Kafka 服务器地址
        value_serializer=lambda v: json.dumps(v).encode('utf-8')  # 序列化为 JSON
    )
    return producer
-----------------------------------------------------------------------------------------------------------
def create_consumer():
    consumer = KafkaConsumer(
        'testa',  # 替换为你想要的主题
        bootstrap_servers='192.168.181.61:9092,192.168.181.66:9092,192.168.181.70:9092',  # Kafka 服务器地址
        auto_offset_reset='earliest',  # 从最早的消息开始消费
        value_deserializer=lambda x: json.loads(x.decode('utf-8')),  # 反序列化为 JSON
        group_id='my-group'  # 消费者组
    )
    return consumer

6、暂停kafka connect 任务，并修改 “database.history.kafka.bootstrap.servers”: “192.168.181.61:9092,192.168.181.66:9092,192.168.181.70:9092” 为新节点地址

update_connect_cdc_config.py

# coding: utf-8
 
import requests
import json
import sys
import argparse
from configparser import ConfigParser
 
class kafkaConnect():
    def __init__(self,base_url) -> None:
        self.base_url = base_url
        self.headers = {
            "Content-Type": "application/json",
            "Accept": "application/json"
        }
    def get_all(self):
        api_url = f"{self.base_url}/connectors"
        resp = requests.get(url=api_url,headers=self.headers)
        if resp.status_code == 200:
            connectors = resp.json()
            return connectors
    def get_config(self,connector_name):
        api_url = f"{self.base_url}/connectors/{connector_name}/config"
        resp = requests.get(url=api_url,headers=self.headers)
        if resp.status_code == 200:
            connectors = resp.json()
            return connectors
    def pause(self,connector_name):
        api_url = f"{self.base_url}/connectors/{connector_name}/pause"
        resp = requests.put(url=api_url,headers=self.headers)
        return {"code":resp.status_code,"msg":resp.text}
    def resume(self,connector_name):
        api_url = f"{self.base_url}/connectors/{connector_name}/resume"
        resp = requests.put(url=api_url,headers=self.headers)
        return {"code":resp.status_code,"msg":resp.text}
    def update_config(self,connector_name,config):
        api_url = f"{self.base_url}/connectors/{connector_name}/config"
        resp = requests.put(url=api_url,headers=self.headers,data=config)
        return {"code":resp.status_code,"msg":resp.text}
     
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-t', '--type', dest='type', type=str, default="", help='')
    args = parser.parse_args()
    type = args.type
    if not type:
        print("require parameters: -t")
        sys.exit(1)
    if type == "update":
        kafka_bootstrap_servers = "192.168.181.61:9092,192.168.181.66:9092,192.168.181.70:9092"
        base_url = "http://192.168.181.121:8083"
        client = kafkaConnect(base_url=base_url)
        connectors = client.get_all()
        for connector_name in connectors:
            status = client.pause(connector_name=connector_name)
            if status['code'] == 202:
                connector_config = client.get_config(connector_name=connector_name)
                connector_config['database.history.kafka.bootstrap.servers'] = kafka_bootstrap_servers
                resp = client.update_config(connector_name=connector_name,config=json.dumps(connector_config))
                print(f"任务：{connector_name},更新：{resp['code']}")
    elif type == "resume":
        base_url = "http://192.168.181.61:8083"
        client = kafkaConnect(base_url=base_url)
        connectors = client.get_all()
        for connector_name in connectors:
            status = client.resume(connector_name=connector_name)
            print(status)

python update_connecter_cdc_config.py -t update
任务：cdc_mdata_sdp4_PR04144_XJGX_1,更新：200
任务：cdc_mdata_sdp4_PR04144_XJGX_a1,更新：200

7、下线kafka集群旧节点

#192.168.181.121
[root@181-121 kafka]# ps -ef|grep config/server.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.14
[root@181-14 kafka]# ps -ef|grep config/server.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.49
[root@181-49 kafka]# ps -ef|grep config/server.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9

8、 下线zookeeper集群旧节点

#192.168.181.121
[root@181-121 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.14
[root@181-14 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.49
[root@181-49 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9

9、修改zookeeper集群新节点配置，去除旧节点信息并重启

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# the directory where the snapshot is stored.
dataDir=/data/kafka/data/zookeeper
# the port at which the clients will connect
clientPort=2181
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
tickTime=2000
initLimit=10
syncLimit=5
 
server.4=192.168.181.61:2888:3888
server.5=192.168.181.66:2888:3888
server.6=192.168.181.70:2888:3888
 
 
#192.168.181.61
[root@181-61 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9 && ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.66
[root@181-66 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9 && ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
 
#192.168.181.70
[root@181-70 kafka]# ps -ef|grep config/zookeeper.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9 && ./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

查看zookeeper集群状态

zk_followers 2
zk_synced_followers 2

[root@181-70 kafka]# echo mntr|nc localhost 2181
zk_version      3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf, built on 03/06/2019 16:18 GMT
zk_avg_latency  0
zk_max_latency  2
zk_min_latency  0
zk_packets_received     13
zk_packets_sent 12
zk_num_alive_connections        2
zk_outstanding_requests 0
zk_server_state leader
zk_znode_count  221
zk_watch_count  20
zk_ephemerals_count     4
zk_approximate_data_size        18067
zk_open_file_descriptor_count   119
zk_max_file_descriptor_count    65536
zk_fsync_threshold_exceed_count 0
zk_followers    2
zk_synced_followers     2
zk_pending_syncs        0
zk_last_proposal_size   -1
zk_max_proposal_size    -1
zk_min_proposal_size    -1

10、在新节点启动kafka connect,停止旧节点kafka connect

[root@181-70 kafka]# cat config/connect-distributed.properties |grep -v '#'
 
 
bootstrap.servers=192.168.181.66:9092,192.168.181.70:9092,192.168.181.61:9092
 
group.id=connect-cluster
 
key.converter=org.apache.kafka.connect.json.JsonConverter
value.converter=org.apache.kafka.connect.json.JsonConverter
key.converter.schemas.enable=true
value.converter.schemas.enable=true
 
offset.storage.topic=connect-offsets
offset.storage.replication.factor=1
 
config.storage.topic=connect-configs
config.storage.replication.factor=1
 
status.storage.topic=connect-status
status.storage.replication.factor=1
 
offset.flush.interval.ms=10000
 
rest.host.name=192.168.181.70
rest.port=8083
 
 
plugin.path=/data/kafka/plugins
errors.retry.timeout=-1
errors.retry.delay.max.ms=120000

#192.168.181.61
[root@181-61 kafka]# ./bin/connect-distributed.sh -daemon ./config/connect-distributed.properties
 
#192.168.181.66
[root@181-66 kafka]# ./bin/connect-distributed.sh -daemon ./config/connect-distributed.properties
 
#192.168.181.70
[root@181-70 kafka]# ./bin/connect-distributed.sh -daemon ./config/connect-distributed.properties
 
#192.168.181.121
[root@181-121 kafka]# ps -ef|grep config/connect-distributed.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.14
[root@181-14 kafka]# ps -ef|grep config/connect-distributed.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9
 
#192.168.181.49
[root@181-49 kafka]# ps -ef|grep config/connect-distributed.properties |grep -v grep|awk -F' ' '{print $2}'|xargs kill -9

11、恢复connect 任务

python update_connecter_cdc_config.py -t resume
{'code': 202, 'msg': ''}
{'code': 202, 'msg': ''}

至此迁移流程完成

注：全程查看producer.py consumer.py运行状态，是否有执行失败