单机搭建zookeeper+kafka&集群搭建&容器搭建

单机搭建zookeeper

下带有bin的包，不需要再进行编译
http://archive.apache.org/dist/zookeeper/zookeeper-3.5.5/apache-zookeeper-3.5.5-bin.tar.gz

[root@vm opt]# tar xf apache-zookeeper-3.5.5-bin.tar.gz
[root@vm opt]# cd apache-zookeeper-3.5.5-bin
[root@vm apache-zookeeper-3.5.5-bin]# ls
bin  conf  docs  lib  LICENSE.txt  NOTICE.txt  README.md  README_packaging.txt
[root@vm apache-zookeeper-3.5.5-bin]# cp conf/zoo_sample.cfg conf/zoo.cfg
[root@vm apache-zookeeper-3.5.5-bin]# mkdir data
[root@vm apache-zookeeper-3.5.5-bin]# vim conf/zoo.cfg
....
dataDir=/root/opt/apache-zookeeper-3.5.5-bin/data
[root@vm apache-zookeeper-3.5.5-bin]# ./bin/zkServer.sh start
[root@vm apache-zookeeper-3.5.5-bin]# ss -antup | grep 2181
tcp    LISTEN     0      50     [::]:2181               [::]:*   users:(("java",pid=

单机搭建kafka

https://archive.apache.org/dist/kafka/
https://archive.apache.org/dist/kafka/2.4.1/kafka_2.11-2.4.1.tgz

[root@vm opt]# tar xf kafka_2.11-2.4.1.tgz
[root@vm opt]# cd kafka_2.11-2.4.1
[root@vm kafka_2.11-2.4.1]# ls
bin  config  libs  LICENSE  NOTICE  site-docs

#配置修改
[root@vm kafka_2.11-2.4.1]# mkdir {data,log}
[root@vm kafka_2.11-2.4.1]# vim config/zookeeper.properties
dataDir=/root/opt/kafka_2.11-2.4.1/data		#修改
clientPort=2181
maxClientCnxns=100
tickTimes=2000
initLimit=10
syncLimit=5
admin.enableServer=false

[root@vm kafka_2.11-2.4.1]# vim config/server.properties
...
broker.id=0 	#每台不同
port=9092 		#端口号默认9092 
host.name=localhost 	#单机可直接用localhost
log.dirs=/root/opt/kafka_2.11-2.4.1/log #日志存放路径可修改可不修改
zookeeper.connect=localhost:2181		#单机这样写

#启动kafka
[root@vm kafka_2.11-2.4.1]# ./bin/kafka-server-start.sh -daemon config/server.properties 
[root@vm kafka_2.11-2.4.1]# ss -antup | grep 9092
tcp    LISTEN     0      50     [::]:9092               [::]:*      users:(("java",
#创建topic
[root@vm kafka_2.11-2.4.1]# ./bin/kafka-topics.sh --zookeeper localhost:2181 --create --replication-factor 1 --partitions 1 --topic  first
Created topic first.
[root@vm kafka_2.11-2.4.1]# ./bin/kafka-topics.sh --zookeeper localhost:2181  --list
first
[root@vm kafka_2.11-2.4.1]# ./bin/kafka-server-stop.sh -daemon config/server.properties

单台服务器上部署集群zookeeper集群

下载地址：http://archive.apache.org/dist/zookeeper/
http://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz

[root@vm opt]# tar xf zookeeper-3.4.14.tar.gz
[root@vm opt]# cd  zookeeper-3.4.14
[root@vm zookeeper-3.4.14]# ls conf/
configuration.xsl  log4j.properties  zoo_sample.cfg
[root@vm zookeeper-3.4.14]# cp conf/zoo_sample.cfg conf/zoo.cfg
[root@vm zookeeper-3.4.14]# mkdir data
[root@vm opt]# cp -r zookeeper-3.4.14 1-zk
[root@vm opt]# cp -r zookeeper-3.4.14 2-zk
[root@vm opt]# cp -r zookeeper-3.4.14 3-zk

#安装配置
[root@vm opt]# vim 1-zk/conf/zoo.cfg		#需要修改处
...
dataDir=/opt/1-zk/data			#目录修改	
clientPort=2181			#端口2-zk用2182  3-zk用2183
...
server.1=192.168.56.10:2888:3888		#多主机搭建不需要改端口，只改IP即可		
server.2=192.168.56.10:2889:3889
server.3=192.168.56.10:2890:3890

server.A=B:C:D
A是一个数字，表示这个是第几号服务器。集群模式下需要在zoo.cfg中dataDir指定的目录下创建
一个文件myid，这个文件里面有一个数据就是A的值，Zookeeper启动时读取此文件，拿到里面的
数据与zoo.cfg里面的配置信息比较从而判断到底是哪个server。 B是这个服务器的地址。
C是这个服务器Follower与集群中的Leader服务器交换信息的端口。
D是万一集群中的Leader服务器挂了，需要一个端口来重新进行选举，选出一个新的Leader，而这
个端口就是用来执行选举时服务器相互通信的端口。　


[root@vm opt]# vim 2-zk/conf/zoo.cfg
...
clientPort=2182....

#创建myid文件
[root@vm opt]# echo 1 >./1-zk/data/myid
[root@vm opt]# echo 2  > 2-zk/data/myid
[root@vm opt]# echo 3  > 3-zk/data/myid

#服务启动
[root@vm opt]# ./1-zk/bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /opt/1-zk/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

[root@vm opt]# ./2-zk/bin/zkServer.sh start
[root@vm opt]# ./3-zk/bin/zkServer.sh start

#状态查看
[root@vm opt]# ./1-zk/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/1-zk/bin/../conf/zoo.cfg
Mode: follower

[root@vm opt]# ./2-zk/bin/zkServer.sh status
Mode: leader
[root@vm opt]# ./3-zk/bin/zkServer.sh status
Mode: follower

[root@vm opt]# ./1-zk/bin/zkCli.sh -server 192.168.56.10:2181
[zk: 192.168.56.10:2181(CONNECTED) 1] ls /
[zookeeper]
[zk: 192.168.56.10:2181(CONNECTED) 3] ls /zookeeper
[quota]
[zk: 192.168.56.10:2181(CONNECTED) 2] get /zookeeper

cZxid = 0x0
ctime = Thu Jan 01 08:00:00 CST 1970
mZxid = 0x0
mtime = Thu Jan 01 08:00:00 CST 1970
pZxid = 0x0
cversion = -1
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 0
numChildren = 1

容器部署zookeeper集群

version: '3'
# 配置zk集群的
# container services下的每一个子配置都对应一个zk节点的docker container
services:
  zoo1:
    image: zookeeper
    hostname: zoo1
    container_name: zoo1
    privileged: true
    ports:              # 配置docker container和宿主机的端口映射
      - "2181:2181"
    environment:                 # 配置docker container的环境变量
      TZ: Asia/Shanghai
      ZOO_MY_ID: 1              # 当前zk实例的id
      # 整个zk集群的机器、端口列表
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
      ZOO_TICK_TIME: 20000      #超时时间
      4lw.commands.whitelist: "*"
    volumes:
      - ./zk1/data:/data                        #数据挂载
      - ./zk1/datalog:/datalog

  zoo2:
    image: zookeeper
    hostname: zoo2
    container_name: zoo2
    privileged: true
    ports:
      - "2182:2181"
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zoo3:2888:3888;2181
      ZOO_TICK_TIME: 20000
      4lw.commands.whitelist: "*"
    volumes:
      - ./zk2/data:/data
      - ./zk2/datalog:/datalog

  zoo3:
    image: zookeeper
    hostname: zoo3
    container_name: zoo3
    privileged: true
    ports:
      - "2183:2181"
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181
      ZOO_TICK_TIME: 20000
      4lw.commands.whitelist: "*"
    volumes:
      - ./zk3/data:/data
      - ./zk3/datalog:/datalog
      
[root@vm zk]# docker exec -it zoo1 bash
root@zoo1:/apache-zookeeper-3.5.9-bin# ./bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

单台服务器上部署集群kafka集群

[root@vm opt]# tar xf kafka_2.11-2.4.1.tgz
[root@vm opt]# cp -r /root/opt/kafka_2.11-2.4.1 1kafka
[root@vm opt]# cp -r /root/opt/kafka_2.11-2.4.1 2kafka
[root@vm opt]# cp -r /root/opt/kafka_2.11-2.4.1 3kafka

#修改每台配置，以1kafka为例，每台都要改
[root@vm opt]# vim 1kafka/config/server.properties
...
broker.id=1		#唯一
port=9092		#唯一
log.dirs=/root/opt/1kafka/logs	#位置修改
zookeeper.connect=localhost:2181,localhost:2182,localhost:2183  #每台一样

#启动kafka，启动方式一样，注意指定正确位置的配置文件
[root@vm opt]#  ./1kafka/bin/kafka-server-start.sh -daemon 1kafka/config/server.properties
[root@vm opt]# ss -antup | grep 909
tcp    LISTEN     0      50     [::]:9092               [::]:*                   users:
tcp    LISTEN     0      50     [::]:9093               [::]:*                   users:
tcp    LISTEN     0      50     [::]:9094               [::]:*                   users:

#创建一个3副本的topic验证，#--replication-factor 定义副本数，不超过节点数
[root@vm opt]# ./1kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --replication-factor 3 --partitions 1 --topic  first
Created topic first.
[root@vm opt]# ./1kafka/bin/kafka-topics.sh --zookeeper  localhost:2181 --list
first
[root@vm opt]# ls 1kafka/logs/
first-0

容器部署zookeeper+kafka

[root@vm kafka]# cat docker-compose.yaml
version: "3"
services:
  zookeeper:
    image: zookeeper:3.5-temurin
    ports:
      - '2181:2181'
    environment:
      TZ: Asia/Shanghai
  kafka:
    image: 'bitnami/kafka:3.1'
    ports:
      - '9092:9092'
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper
      
[root@vm kafka]# docker-compose up -d
[root@vm ~]# docker exec -it 9fb05954f8db bash

I have no name!@9fb05954f8db:/$ cd /opt/bitnami/kafka/bin
I have no name!@9fb05954f8db:/opt/bitnami/kafka/bin$ ls
I have no name!@9fb05954f8db:/opt/bitnami/kafka/bin$ kafka-topics.sh --create --bootstrap-server localhost:9092 --topic mytopic
Created topic mytopic.
I have no name!@9fb05954f8db:/opt/bitnami/kafka/bin$ kafka-topics.sh  --bootstrap-server localhost:9092 --list
mytopic

Zookeeper数据可视化查看工具

ZooInspectorhttps://issues.apache.org/jira/secure/attachment/12436620/ZooInspector.zip
windows安装java启动
cd /d d:\Downloads\ZooInspector\build
java -jar zookeeper-dev-ZooInspector.jar