0x01 Redis集群安装
集群需要6个redis服务,所以, 计划在三台主机上, 分别开启 6379, 6380 两个服务
# 在三台机器分别操作:
cd /app/soft
wget http://download.redis.io/releases/redis-5.0.7.tar.gz
tar -zxf redis-5.0.7.tar.gz
cd redis-5.0.7
make
mkdir /app/redis
cd /app/soft/redis-5.0.7/src
cp redis-cli redis-server redis-check-aof redis-sentinel redis-check-rdb redis-trib.rb mkreleasehdr.sh /app/redis
cd /app/soft/redis-5.0.7
cp redis.conf /app/redis
mkdir /app/redis/data-6379
mkdir /app/redis/data-6380
- 修改 redis-6379.conf
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /app/redis/data-6379/redis.log
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /app/redis/data-6379
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
cluster-enabled yes
cluster-node-timeout 15000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
- 修改 redis-6380.conf
protected-mode no
port 6380
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6380.pid
loglevel notice
logfile /app/redis/data-6380/redis.log
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /app/redis/data-6380
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
cluster-enabled yes
cluster-node-timeout 15000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
- 启动 redis
cd /app/redis
./redis-server redis-6379.conf
./redis-server redis-6380.conf
- 开启集群
cd /app/redis
./redis-cli --cluster create 172.16.36.230:6379 172.16.36.230:6380 172.16.36.228:6379 172.16.36.228:6380 172.16.36.227:6379 172.16.36.227:6380 --cluster-replicas 1
创建集群过程
到此, 三主三从集群创建完成
- 登录
./redis-cli -c -h 172.16.36.230 -p 6379
# 查询集群信息
> CLUSTER NODES
> CLUSTER INFO
0x02 zookeeper 三机集群安装
cd /app/soft
wget https://archive.apache.org/dist/zookeeper/zookeeper-3.5.6/apache-zookeeper-3.5.6-bin.tar.gz
tar -zxvf apache-zookeeper-3.5.6-bin.tar.gz
ln -s /app/soft/apache-zookeeper-3.5.6-bin /app/zookeeper
cd /app/zookeeper/conf
cp zoo_sample.cfg zoo.cfg
- 编辑 zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/zookeeper/data
clientPort=2181
server.1=172.16.36.230:2888:3888
server.2=172.16.36.227:2888:3888
server.3=172.16.36.228:2888:3888
- 配置 myid
mkdir /app/zookeeper/data
cd /app/zookeeper/data
touch myid
# 分别写上 1,2,3
- 启动&停止
# 三个服务器,分别执行
cd /app/zookeeper
./bin/zkServer.sh start
./bin/zkServer.sh stop
- 状态确认:
cd /app/zookeeper
./bin/zkServer.sh status
image.png
0x03 Kafka集群安装
cd /app/soft
wget http://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.3.0/kafka_2.12-2.3.0.tgz
tar zxf kafka_2.12-2.3.0.tgz
ln -s /app/soft/kafka_2.12-2.3.0 /app/kafka
mkdir /app/kafka/kakfa-logs
- 编辑 config/server.properties
broker.id=1 # 每台机器不一样, 分别为 1,2,3
host.name=172.16.36.230 # 每台机器不一样, 都为本机IP
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/app/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.retention.bytes=10737418240
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.16.36.230:2181,172.16.36.228:2181,172.16.36.227:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
- 启动服务
cd /app/kafka
./bin/kafka-server-start.sh -daemon ./config/server.properties
- 关闭服务
cd /app/kafka
./bin/kafka-server-stop.sh
- 用于公网访问的集群配置:
broker.id=1
advertised.listeners=PLAINTEXT://xx.xx.xx.xx:9092 # 这里写公网IP
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/app/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.retention.bytes=10737418240
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.16.36.230:2181,172.16.36.228:2181,172.16.36.227:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
集群的几台主机之间也需要相互开放
9092端口访问
- 常用命令行工具:
# 内网消费者
kafka-console-consumer.sh --bootstrap-server 172.16.36.230:9092,172.16.36.228:9092,172.16.36.227:9092 --topic my-kafka-topic
# 内网生产者
kafka-console-producer.sh --broker-list 172.16.36.230:9092,172.16.36.228:9092,172.16.36.227:9092 --topic my-kafka-topic