Kafka集群搭建

Java环境

mkdir -p /opt/archives/jdk
tar -xzv -f jdk-7u80-linux-x64.tar.gz -C /opt/archives/jdk/
cd /opt/
ln -s /opt/archives/jdk/jdk1.7.0_80/ jdk

vi ~/.bashrc
# 在文件结尾加上这三行
JAVA_HOME=/opt/jdk
CLASSPATH=.:$JAVA_HOME/lib
PATH=$JAVA_HOME/bin:$PATH

source ~/.bashrc
java -version

zookeeper集群

mkdir -p /opt/archives/zookeeper
tar -xzv -f zookeeper-3.4.6.tar.gz -C /opt/archives/zookeeper/
ln -s /opt/archives/zookeeper/zookeeper-3.4.6/ /opt/zookeeper

# 创建要用到的目录
cd /opt
mkdir zkdata
mkdir zkdatalog

# 修改配置文件
cd /opt/zookeeper/conf
cp zoo_sample.cfg zoo.cfg
vi zoo.cfg
# 修改配置如下图所示

# 创建标识文件
cd /opt/zkdata
echo 1 > myid

image.png

# 当前操作的主机192.168.234.130,还有其他两台主机192.168.234.131,192.168.234.132
# 把130机器上的opt目录复制到131和132
scp -r /opt/archives [email protected]:/opt/
scp -r /opt/archives [email protected]:/opt/
# 创建链接 cd
ln -s /opt/archives/jdk/jdk1.7.0_80/ /opt/jdk
ln -s /opt/archives/zookeeper/zookeeper-3.4.6/ /opt/zookeeper

# 分别修改131和132上的Java配置和zookeeper配置
# ~/.bashrc文件，添加java配置
# /opt/zkdata/myid，配置每个zookeeper的id


# 启动前要关闭防火墙或配置防火墙
# 这里先把防火墙关闭了
service iptables stop
chkconfig iptables off
service iptables status

# 启动zookeeper
cd /opt/zookeeper
bin/zkServer.sh start
# 只有每台zookeeper都起来，过一会再查看zookeeper状态才正常
bin/zkServer.sh status
# 关闭zookeeper
bin/zkServer.sh stop
# 两个follower一个leader
用jps查看进程

Kafka集群

mkdir -p /opt/archives/kafka
tar -xzv -f kafka_2.11-0.10.2.1.tgz -C /opt/archives/kafka/
ln -s /opt/archives/kafka/kafka_2.11-0.10.2.1/ /opt/kafka

mkdir /opt/kafkalogs

vi /opt/kafka/config/server.properties

image.png

追加如下配置

message.max.bytes=1000012
The maximum size of message that the server can receive(每条消息最大字节数，默认是1000012)

default.replication.factor=1
default replication factors for automatically created topics(默认复制因子，保存消息的副本数)

replica.fetch.max.bytes=1048576
取消息的最大字节数，默认1048576

改成如下
message.max.bytes=5242880
default.replication.factor=2
replica.fetch.max.bytes=5242880

sed -i '$a message.max.bytes=5242880\ndefault.replication.factor=2\nreplica.fetch.max.bytes=5242880' server.properties

修改消息存放目录
修改连接地址

最终修改结果

image.png

配置其他两台机器

scp -r /opt/archives/kafka/ [email protected]:/opt/archives/
scp -r /opt/archives/kafka/ [email protected]:/opt/archives/

在131，132这两台机子上执行
cd /opt/
ln -s archives/kafka/kafka_2.11-0.10.2.1/ kafka
mkdir kafkalogs
vi kafka/config/server.properties
主要修改broker.id
advertised.host.name

启动
bin/kafka-server-start.sh -daemon config/server.properties

测试

#创建topic
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 1 --topic test
#查看topic列表
bin/kafka-topics.sh --list --zookeeper localhost:2181
#查看单个topic信息
#创建consumer，用来接收消息
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
#创建producer，用来发送消息
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
#查看主题信息
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test