大数据组件kafka部署安装与测试练习

环境说明

10.176.2.101 master
10.176.2.103 zjx03
10.176.2.105 zjx05

cent-os6.5
zookeeper cdh 3.4.5
hadoop apache 2.7.7
mysql 5.17
jdk 1.8.191
kafka 2.11_2.01

kafka官网

http://kafka.apache.org/

kafka安装

#master
[root@master softwares]# cd /opt/softwares
[root@master softwares]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.0.1/kafka_2.11-2.0.1.tgz -c /opt/softwares/
[root@master softwares]# tar -zxvf kafka_2.11-2.0.1.tgz
[root@master softwares]# cd ./kafka_2.11-2.0.1/config
#10.176.2.101 master节点修改broker.id、listeners、zookeeper.connect
[root@master config]# vim server.properties
broker.id=0
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://master:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://master:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#kafaka复制至其它机器
[root@master softwares]# scp -r ./kafka_2.11-2.0.1 zjx03:/opt/softwares/
[root@master softwares]# scp -r ./kafka_2.11-2.0.1 zjx05:/opt/softwares/
#10.176.2.103 zjx03节点修改broker.id、listeners、zookeeper.connect
[root@zjx03 config]# vim server.properties
broker.id=1
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://zjx03:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://zjx03:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#10.176.2.105 zjx05节点修改broker.id、listeners、zookeeper.connect
[root@zjx05 config]# vim server.properties
broker.id=2
#listeners=PLAINTEXT://:9092
listeners=PLAINTEXT://zjx05:9092
#advertised.listeners=PLAINTEXT://your.host.name:9092
advertised.listeners=PLAINTEXT://zjx05:9092
zookeeper.connect=master:2181,zjx03:2181,zjx05:2181
#配置环境变量
[root@master config]# vim /etc/profile
export KAFKA_HOME=/opt/softwares/kafka_2.11-2.0.1
export PATH=$KAFKA_HOME/bin:$PATH
[root@master config]# source /etc/profile
#另两个节点同样进行环境变量配置

后台开启3台虚拟机kafka程序

cd /opt/softwares/kafka_2.11-2.0.1
./bin/kafka-server-start.sh -daemon config/server.properties
#或者
./bin/kafka-server-start.sh  config/server.properties &

创建topic

可指定Partition、Replication数量
建议--replication-factor > 1，否则Broker宕掉后将无法写入消息

#3个分区3个副本
[root@master bin]# ./kafka-topics.sh --create --topic topic_zjx_test --zookeeper master:2181,zjx03:2181,zjx05:2181 --partitions 3 --replication-factor 3
#3个分区1个副本
[root@master bin]# ./kafka-topics.sh --create --topic topic_zjx  --zookeeper master:2181,zjx03:2181,zjx05:2181 --partitions 3 --replication-factor 1
#查看topic列表
[root@master bin]# ./kafka-topics.sh --zookeeper master:2181,zjx03:2181,zjx05:2181 --list
topic_zjx
topic_zjx_test
#查看topic详情 
[root@master bin]# ./kafka-topics.sh  --describe --topic topic_zjx  --zookeeper master:2181,zjx03:2181,zjx05:2181
Topic:topic_zjx PartitionCount:3    ReplicationFactor:1 Configs:
    Topic: topic_zjx    Partition: 0    Leader: 0   Replicas: 0 Isr: 0
    Topic: topic_zjx    Partition: 1    Leader: 0   Replicas: 0 Isr: 0
    Topic: topic_zjx    Partition: 2    Leader: 0   Replicas: 0 Isr: 0
[root@master bin]# ./kafka-topics.sh  --describe --topic topic_zjx_test  --zookeeper master:2181,zjx03:2181,zjx05:2181
Topic:topic_zjx_test    PartitionCount:3    ReplicationFactor:3 Configs:
    Topic: topic_zjx_test   Partition: 0    Leader: 2   Replicas: 2,0,1 Isr: 2,0,1
    Topic: topic_zjx_test   Partition: 1    Leader: 0   Replicas: 0,1,2 Isr: 0,1,2
    Topic: topic_zjx_test   Partition: 2    Leader: 1   Replicas: 1,2,0 Isr: 1,2,0

创建producer

[root@master config]# kafka-console-producer.sh --broker-list master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test

创建consumer

新开一个session

[root@master ~]# cd /opt/softwares/kafka_2.11-2.0.1/bin
[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test

在producer session中输入，consumer session中会自动接收消息

[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test
>zhoujixiang
>hello kafka
>yudd
[root@master bin]# ./kafka-console-consumer.sh --bootstrap-server master:9092, zjx03:9092, zjx05:9092 --topic topic_zjx_test
zhoujixiang
hello kafka
yudd

停止kafka

kafka-server-stop.sh

参考链接：
https://blog.csdn.net/qq_38270106/article/details/83715910
https://www.cnblogs.com/xyj179/p/10020865.html
https://blog.csdn.net/weixin_38120374/article/details/80608496
kafaka常用命令：
https://www.cnblogs.com/dragkiss/p/5668019.html