Redis 3.0 集群特性实验过程

        Redis 3.0的集群功能很强大了， 最大的特点就是有了cluster的能力，使用redis-trib.rb工具可以轻松构建Redis Cluster。Redis Cluster采用无中心结构，每个节点保存数据和整个集群状态,每个节点都和其他所有节点连接。节点之间使用gossip协议传播信息以及发现新节点。其具体的原理请看官方文档，这里只是记录一下实验的过程。以便更好的了解集群的特性。

#安装依赖软件：

 yum install gcc rubyzlib rubygems

 wgethttps://rubygems.org/downloads/redis-3.2.2.gem

 gem installredis-3.2.2.gem

#版本：redis-3.2

#架构信息

192.168.100.41   master：6379  slave：7379

192.168.100.42   master：6379  slave：7379

192.168.100.43   master：6379  slave：7379

#通用配置：

more /usr/local/redis-3.2/conf/redis-common.conf

#GENERAL

daemonize yes

tcp-backlog 511

timeout 0

tcp-keepalive 0

loglevel notice

databases 16

dir /usr/local/redis-3.2/{data,data_7379}

slave-serve-stale-data yes

slave-read-only yes

#not use default

repl-disable-tcp-nodelay yes

slave-priority 100

appendonly yes

appendfsync everysec

no-appendfsync-on-rewrite yes

auto-aof-rewrite-min-size 64mb

lua-time-limit 5000

cluster-enabled yes

cluster-node-timeout 15000

cluster-migration-barrier 1

slowlog-log-slower-than 10000

slowlog-max-len 128

notify-keyspace-events ""

hash-max-ziplist-entries 512

hash-max-ziplist-value 64

list-max-ziplist-entries 512

list-max-ziplist-value 64

set-max-intset-entries 512

zset-max-ziplist-entries 128

zset-max-ziplist-value 64

activerehashing yes

client-output-buffer-limit normal 0 0 0

client-output-buffer-limit slave 256mb 64mb 60

client-output-buffer-limit pubsub 32mb 8mb 60

hz 10

aof-rewrite-incremental-fsync yes

 

 

#独立端口配置：{6379,7379}

 

more /usr/local/redis-3.2/conf/redis-{6379,7379}.conf

include/usr/local/redis-3.2/conf/redis-{common,common_7379}.conf

port {6379,7379}

logfile"/usr/local/redis-3.2/logs/redis-{6379,7379}.log"

maxmemory 100m

# volatile-lru -> remove the key with an expire set usingan LRU algorithm

# allkeys-lru -> remove any key accordingly to the LRUalgorithm

# volatile-random -> remove a random key with an expireset

# allkeys-random -> remove a random key, any key

# volatile-ttl -> remove the key with the nearest expiretime (minor TTL)

# noeviction -> don't expire at all, just return an erroron write operations

maxmemory-policy allkeys-lru

appendfilename "appendonly-{6379,7379}.aof"

dbfilename dump-{6379,7379}.rdb

cluster-config-file nodes-{6379,7379}.conf

auto-aof-rewrite-percentage 80-100

bind 192.168.100.{41,42,43}

 

#启动进程

/usr/local/redis-3.2/bin/redis-server/usr/local/redis-3.2/conf/redis-{6379,7379}.conf

#创建集群

 

/usr/local/redis-3.2/bin/redis-trib.rb create --replicas 1192.168.100.41:6379 192.168.100.42:6379 192.168.100.43:6379 192.168.100.41:7379192.168.100.42:7379

192.168.100.43:7379

 

>>> Creating cluster

>>> Performing hash slots allocation on 6 nodes...

Using 3 masters:

192.168.100.43:6379

192.168.100.42:6379

192.168.100.41:6379

Adding replica 192.168.100.42:7379 to 192.168.100.43:6379

Adding replica 192.168.100.41:7379 to 192.168.100.42:6379

Adding replica 192.168.100.42:7379 to 192.168.100.41:6379

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd 192.168.100.41:6379

   slots:10923-16383(5461 slots) master

M: 35fc4a46cfe68e941a18ca33e574df86db7beefb192.168.100.42:6379

   slots:5461-10922(5462 slots) master

M: 2ef9b515fac6159b37520afce1f75b38ba1e9a87192.168.100.43:6379

   slots:0-5460 (5461slots) master

S: 6a2d10792f17985d1e30e9e20fe92c890748487f192.168.100.41:7379

   replicates35fc4a46cfe68e941a18ca33e574df86db7beefb

S: eb921729e82925c6be859185efb58e77b49e7a89192.168.100.42:7379

   replicates2ef9b515fac6159b37520afce1f75b38ba1e9a87

S: eb921729e82925c6be859185efb58e77b49e7a89192.168.100.42:7379

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

Can I set the above configuration? (type 'yes' to accept):YES

>>> Nodes configuration updated

>>> Assign a different config epoch to each node

>>> Sending CLUSTER MEET messages to join thecluster

Waiting for the cluster to join.

>>> Performing Cluster Check (using node192.168.100.41:6379)

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:10923-16383(5461 slots) master

M: 35fc4a46cfe68e941a18ca33e574df86db7beefb192.168.100.42:6379

   slots:5461-10922(5462 slots) master

M: 2ef9b515fac6159b37520afce1f75b38ba1e9a87192.168.100.43:6379

   slots:0-5460 (5461slots) master

M: 6a2d10792f17985d1e30e9e20fe92c890748487f 192.168.100.41:7379

   slots: (0 slots)master

   replicates35fc4a46cfe68e941a18ca33e574df86db7beefb

M: eb921729e82925c6be859185efb58e77b49e7a89192.168.100.42:7379

   slots: (0 slots)master

   replicates2ef9b515fac6159b37520afce1f75b38ba1e9a87

M: eb921729e82925c6be859185efb58e77b49e7a89192.168.100.42:7379

   slots: (0 slots)master

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

 

#添加节点

master:192.168.100.41:8379 slave:192.168.100.42:8379 slave:192.168.100.43:8379

创建相关配置文件和数据目录同上述配置更改端口信息即可

 

#节点准备就绪

salt '*' cmd.run ' ps aux |grep redis-server|grep -v grep'

client.wboy.com:

    root     13452 0.0  0.7  36684 7716 ?        Ssl  11:20  0:13 /usr/local/redis-3.2/bin/redis-server 192.168.100.42:7379 [cluster]           

    root     13462 0.1  0.9  38732 9756 ?        Ssl  11:20  0:21 /usr/local/redis-3.2/bin/redis-server 192.168.100.42:6379 [cluster]            

    root     16566 0.0  0.7  36684 7560 ?        Ssl  16:02  0:00 /usr/local/redis-3.2/bin/redis-server 192.168.100.42:8379 [cluster]

master.weiboyi.com:

    root      3991 0.1  0.5  40780 9800 ?        Ssl  11:20  0:22 /usr/local/redis-3.2/bin/redis-server 192.168.100.41:6379[cluster]           

    root      4324 0.0  0.4 133520  7720 ?       Ssl  11:27   0:12 /usr/local/redis-3.2/bin/redis-server192.168.100.41:7379 [cluster]           

    root     16347 0.0  0.3  36684 7560 ?        Ssl 16:02   0:00/usr/local/redis-3.2/bin/redis-server 192.168.100.41:8379 [cluster]

client1.weiboyi.com:

    root     10014 0.1  0.3  36684 7736 ?        Ssl  12:19  0:31 /usr/local/redis-3.2/bin/redis-server 192.168.100.43:7379 [cluster]            

    root     10027 0.2  0.4  38732 9788 ?        Ssl  12:20  0:39 /usr/local/redis-3.2/bin/redis-server 192.168.100.43:6379 [cluster]           

    root     13153 0.0  0.3  36684 7564 ?        Ssl  17:02  0:00 /usr/local/redis-3.2/bin/redis-server 192.168.100.43:8379 [cluster]

 

#先检查一下节点的信息状态等

/usr/local/redis-3.2/bin/redis-trib.rb check192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:9223-16383(7161 slots) master

   1 additionalreplica(s)

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a 192.168.100.43:6379

  slots:0-1879,5461-9222 (5642 slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af192.168.100.42:6379

   slots:1880-5460(3581 slots) master

   1 additionalreplica(s)

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

#加入节点

 

/usr/local/redis-3.2/bin/redis-trib.rb add-node192.168.100.41:8379 192.168.100.41:7379

                                                 

>>> Adding node 192.168.100.41:8379 to cluster192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:9223-16383(7161 slots) master

   1 additionalreplica(s)

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

   slots:0-1879,5461-9222(5642 slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af 192.168.100.42:6379

   slots:1880-5460(3581 slots) master

   1 additionalreplica(s)

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

>>> Send CLUSTER MEET to node 192.168.100.41:8379to make it join the cluster.

[OK] New node added correctly.

 

#check 加入节点后的集群信息

 /usr/local/redis-3.2/bin/redis-trib.rbcheck 192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

M: 2c332aaea894f479c462f6d3eaac580b4833babd192.168.100.41:8379

   slots: (0 slots)master

   0 additionalreplica(s)

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:9223-16383(7161 slots) master

   1 additional replica(s)

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

  slots:0-1879,5461-9222 (5642 slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af192.168.100.42:6379

   slots:1880-5460(3581 slots) master

   1 additionalreplica(s)

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

 

#继续添加192.168.100.42:8379  192.168.100.43:8379

#设置主从

/usr/local/redis-3.2/bin/redis-cli -c -h 192.168.100.42 -p8379

192.168.100.42:8379>

192.168.100.42:8379> cluster replicate2c332aaea894f479c462f6d3eaac580b4833babd

OK

192.168.100.42:8379>

/usr/local/redis-3.2/bin/redis-cli -c -h 192.168.100.43 -p8379

192.168.100.43:8379> cluster replicate2c332aaea894f479c462f6d3eaac580b4833babd

OK

 

 

/usr/local/redis-3.2/bin/redis-trib.rb check192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56 192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

  slots:0-1879,5461-9222 (5642 slots) master

   1 additionalreplica(s)

S: a2632a65e72f04eb4e3b7cda228908574438e1cb192.168.100.42:8379

   slots: (0 slots)slave

   replicates2c332aaea894f479c462f6d3eaac580b4833babd

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

M: 2c332aaea894f479c462f6d3eaac580b4833babd192.168.100.41:8379

   slots: (0 slots)master

   2 additionalreplica(s)

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:9223-16383(7161 slots) master

   1 additionalreplica(s)

S: 744ee15ad1ec0ee98bf9a94c24262cd9242e7105192.168.100.43:8379

   slots: (0 slots)slave

   replicates2c332aaea894f479c462f6d3eaac580b4833babd

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af192.168.100.42:6379

   slots:1880-5460 (3581slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

#重新平衡slot

/usr/local/redis-3.2/bin/redis-trib.rb reshard 192.168.100.41:7379

 

Moving slot 10229 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10230 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10231 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10232 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10233 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10234 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10235 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10236 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10237 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10238 from 192.168.100.41:6379 to192.168.100.42:6379:

Moving slot 10239 from 192.168.100.41:6379 to192.168.100.42:6379:

 

#删除节点

 /usr/local/redis-3.2/bin/redis-trib.rbdel-node 192.168.100.41:8379 2c332aaea894f479c462f6d3eaac580b4833babd

>>> Removing node2c332aaea894f479c462f6d3eaac580b4833babd from cluster 192.168.100.41:8379

>>> Sending CLUSTER FORGET messages to thecluster...

>>> SHUTDOWN the node.

 

 /usr/local/redis-3.2/bin/redis-trib.rbdel-node 192.168.100.42:8379 a2632a65e72f04eb4e3b7cda228908574438e1cb

>>> Removing nodea2632a65e72f04eb4e3b7cda228908574438e1cb from cluster 192.168.100.42:8379

>>> Sending CLUSTER FORGET messages to thecluster...

>>> SHUTDOWN the node.

 /usr/local/redis-3.2/bin/redis-trib.rbdel-node 192.168.100.43:8379 744ee15ad1ec0ee98bf9a94c24262cd9242e7105

>>> Removing node744ee15ad1ec0ee98bf9a94c24262cd9242e7105 from cluster 192.168.100.43:8379

>>> Sending CLUSTER FORGET messages to thecluster...

>>> SHUTDOWN the node.

 

#failover 关闭主节点192.168.100.418379

[root@master conf]# redis-cli -h  192.168.100.41 -p 8379 debug segfault

Error: Server closed the connection

[root@master conf]# /usr/local/redis-3.2/bin/redis-trib.rbcheck 192.168.100.41:7379

[ERR] Sorry, can't connect to node 192.168.100.41:8379

*** WARNING: 192.168.100.43:8379 claims to be slave ofunknown node ID c7c4a1fb3d86f047276385b5695aaca1e1137818.

*** WARNING: 192.168.100.42:8379 claims to be slave ofunknown node ID c7c4a1fb3d86f047276385b5695aaca1e1137818.

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

   slots:1250-5119(3870 slots) master

   1 additionalreplica(s)

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicates c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

S: d25b0f21422628ca876459675895b0ca6e5cb3ec192.168.100.43:8379

   slots: (0 slots)slave

   replicatesc7c4a1fb3d86f047276385b5695aaca1e1137818

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:11740-16383(4644 slots) master

   1 additionalreplica(s)

S: 96dbe347b4ceeb419005ab1a3f98a11c7821ff07192.168.100.42:8379

   slots: (0 slots)slave

   replicatesc7c4a1fb3d86f047276385b5695aaca1e1137818

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af 192.168.100.42:6379

   slots:6370-10239(3870 slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[ERR] Not all 16384 slots are covered by nodes.

 

#M: 96dbe347b4ceeb419005ab1a3f98a11c7821ff07192.168.100.42:8379  变成主节点

 

[root@master conf]# /usr/local/redis-3.2/bin/redis-trib.rbcheck 192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

   slots:1250-5119(3870 slots) master

   1 additionalreplica(s)

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

S: d25b0f21422628ca876459675895b0ca6e5cb3ec192.168.100.43:8379

   slots: (0 slots)slave

   replicates96dbe347b4ceeb419005ab1a3f98a11c7821ff07

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:11740-16383(4644 slots) master

   1 additionalreplica(s)

M: 96dbe347b4ceeb419005ab1a3f98a11c7821ff07192.168.100.42:8379

  slots:0-1249,5120-6369,10240-11739 (4000 slots) master

   1 additionalreplica(s)

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af192.168.100.42:6379

   slots:6370-10239 (3870slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

#启动失败的主节点，自动变成从节点

 

[root@master conf]# salt 'master.wboy.cn' cmd.run  '/usr/local/redis-3.2/bin/redis-server/usr/local/redis-3.2/conf/redis-8379.conf'

master.wboy.com:

[root@master conf]# /usr/local/redis-3.2/bin/redis-trib.rbcheck 192.168.100.41:7379

>>> Performing Cluster Check (using node192.168.100.41:7379)

S: 589ff9053237d77131f4cc6f6cf0006b3e38ea56192.168.100.41:7379

   slots: (0 slots)slave

   replicatesc10928731b4563ec3aa7a60c6837ee77b8b0f95a

M: c10928731b4563ec3aa7a60c6837ee77b8b0f95a192.168.100.43:6379

   slots:1250-5119(3870 slots) master

   1 additionalreplica(s)

S: c7c4a1fb3d86f047276385b5695aaca1e1137818192.168.100.41:8379

   slots: (0 slots) slave

   replicates96dbe347b4ceeb419005ab1a3f98a11c7821ff07

S: fec421bd36ad2730864d520240c77248494881cd192.168.100.42:7379

   slots: (0 slots)slave

   replicatesc2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd

S: d25b0f21422628ca876459675895b0ca6e5cb3ec 192.168.100.43:8379

   slots: (0 slots)slave

   replicates96dbe347b4ceeb419005ab1a3f98a11c7821ff07

M: c2b3c9cb4b040e4ce48c7a20b4000a1d02e674bd192.168.100.41:6379

   slots:11740-16383(4644 slots) master

   1 additionalreplica(s)

M: 96dbe347b4ceeb419005ab1a3f98a11c7821ff07192.168.100.42:8379

  slots:0-1249,5120-6369,10240-11739 (4000 slots) master

   2 additionalreplica(s)

M: 389e7e2fa0a4fb494f6ecb854a677d04e6ed79af192.168.100.42:6379

   slots:6370-10239(3870 slots) master

   1 additionalreplica(s)

S: dba4b4dca703cd2776e2771f8805139a2beaa306192.168.100.43:7379

   slots: (0 slots)slave

   replicates389e7e2fa0a4fb494f6ecb854a677d04e6ed79af

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.