redis源码之sentinel高可用架构分析-分布式一致性Raft算法
=====================================================
redis源码学习系列文章：
redis源码分析之sha1算法分析
redis源码分析之字典源码分析
redis源码分析之内存编码分析intset, ziplist编码分析
redis源码分析之跳跃表
redis源码分析之内存淘汰策略的原理分析
redis源码分析之对象系统源码分析string, list链表,hash哈希,set集合,zset有序集合
redis源码分析之异步进程保存数据rdb文件和aof文件源码分析
redis源码之sentinel高可用架构分析
=====================================================
在我的github上会持续更新Redis代码的中文分析，地址送出https://github.com/chensongpoixs/credis_source，共同学习进步
前言
redis中高可用模型 ：使用Raft一致性算法比较简单,
正文
1, 整体一个流程图
2, sentinel与master,slave服务的通信
sentinel服务主要存在在定时器sentinelTimer方法中执行的的流程
sentinelHandleDictOfRedisInstances->sentinelHandleRedisInstance->sentinelReconnectInstance
①,连接master
在sentinelReconnectInstance方法找那个连接master服务器并且订阅"sentinel:hello"频道
/* Create the async connections for the instance link if the link
* is disconnected. Note that link->disconnected is true even if just
* one of the two links (commands and pub/sub) is missing. */
/**
* sentinel 异步连接master 服务的操作 和订阅hello的操作
* @param ri masters服务器的的信息
*/
void sentinelReconnectInstance(sentinelRedisInstance *ri) {
if (ri->link->disconnected == 0) return;
if (ri->addr->port == 0) return; /* port == 0 means invalid address. */
instanceLink *link = ri->link;
mstime_t now = mstime();
//从连的时间的秒数
if (now - ri->link->last_reconn_time < SENTINEL_PING_PERIOD)
{
return;
}
ri->link->last_reconn_time = now;
/* Commands connection. */
if (link->cc == NULL) {
link->cc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR);
if (link->cc->err) {
sentinelEvent(LL_DEBUG,"-cmd-link-reconnection",ri,"%@ #%s", link->cc->errstr);
instanceLinkCloseConnection(link,link->cc);
} else {
link->pending_commands = 0;
link->cc_conn_time = mstime();
link->cc->data = link;
// 设置回调函数处理
// 这里相当于 net init的操作 --> 现在要找到startup net的函数
redisAeAttach(server.el,link->cc);
//连接上回函数 注册写入事件 hiredis
redisAsyncSetConnectCallback(link->cc, sentinelLinkEstablishedCallback);
redisAsyncSetDisconnectCallback(link->cc, sentinelDisconnectCallback);
sentinelSendAuthIfNeeded(ri,link->cc);
sentinelSetClientName(ri,link->cc,"cmd");
/* Send a PING ASAP when reconnecting. */
sentinelSendPing(ri);
}
}
/* Pub / Sub */
if ((ri->flags & (SRI_MASTER|SRI_SLAVE)) && link->pc == NULL) {
link->pc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR);
if (link->pc->err) {
sentinelEvent(LL_DEBUG,"-pubsub-link-reconnection",ri,"%@ #%s",
link->pc->errstr);
instanceLinkCloseConnection(link,link->pc);
} else {
int retval;
link->pc_conn_time = mstime();
link->pc->data = link;
redisAeAttach(server.el,link->pc);
// 身份验证
redisAsyncSetConnectCallback(link->pc, sentinelLinkEstablishedCallback);
redisAsyncSetDisconnectCallback(link->pc, sentinelDisconnectCallback);
sentinelSendAuthIfNeeded(ri,link->pc);
sentinelSetClientName(ri,link->pc,"pubsub");
/* Now we subscribe to the Sentinels "Hello" channel. */
// 订阅 hello的操作
retval = redisAsyncCommand(link->pc, sentinelReceiveHelloMessages, ri, "%s %s", sentinelInstanceMapCommand(ri,"SUBSCRIBE"), SENTINEL_HELLO_CHANNEL);
if (retval != C_OK) {
/* If we can't subscribe, the Pub/Sub connection is useless
* and we can simply disconnect it and try again. */
instanceLinkCloseConnection(link,link->pc);
return;
}
}
}
/* Clear the disconnected status only if we have both the connections
* (or just the commands connection if this is a sentinel instance). */
if (link->cc && (ri->flags & SRI_SENTINEL || link->pc))
link->disconnected = 0;
}
② 获取master服务下属从服务salve服务的信息
sentinelSendPeriodicCommands方法中发送info命令获取从服务的基本的信息返回信息连接下属从服务函数sentinelRefreshInstanceInfo方法
/* Send periodic PING, INFO, and PUBLISH to the Hello channel to
* the specified master or slave instance. */
/**
* sentinel 服务的推送信息的处理 master slave 操作
* sentinel服务回去master的的信息 info的操作
* @param ri
*/
void sentinelSendPeriodicCommands(sentinelRedisInstance *ri) {
mstime_t now = mstime();
mstime_t info_period, ping_period;
int retval;
/* Return ASAP if we have already a PING or INFO already pending, or
* in the case the instance is not properly connected. */
if (ri->link->disconnected)
{
return;
}
/* For INFO, PING, PUBLISH that are not critical commands to send we
* also have a limit of SENTINEL_MAX_PENDING_COMMANDS. We don't
* want to use a lot of memory just because a link is not working
* properly (note that anyway there is a redundant protection about this,
* that is, the link will be disconnected and reconnected if a long
* timeout condition is detected. */
if (ri->link->pending_commands >= SENTINEL_MAX_PENDING_COMMANDS * ri->link->refcount)
{
return;
}
/* If this is a slave of a master in O_DOWN condition we start sending
* it INFO every second, instead of the usual SENTINEL_INFO_PERIOD
* period. In this state we want to closely monitor slaves in case they
* are turned into masters by another Sentinel, or by the sysadmin.
*
* Similarly we monitor the INFO output more often if the slave reports
* to be disconnected from the master, so that we can have a fresh
* disconnection time figure. */
if ((ri->flags & SRI_SLAVE) && ((ri->master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS)) || (ri->master_link_down_time != 0)))
{
info_period = 1000;
}
else
{
info_period = SENTINEL_INFO_PERIOD;
}
/* We ping instances every time the last received pong is older than
* the configured 'down-after-milliseconds' time, but every second
* anyway if 'down-after-milliseconds' is greater than 1 second. */
ping_period = ri->down_after_period;
if (ping_period > SENTINEL_PING_PERIOD)
{
ping_period = SENTINEL_PING_PERIOD;
}
//redis中处理write回调函数使用列表维护的了 需要我们可以 使用是有序的处理的
/* Send INFO to masters and slaves, not sentinels. */
if ((ri->flags & SRI_SENTINEL) == 0 && (ri->info_refresh == 0 || (now - ri->info_refresh) > info_period))
{
retval = redisAsyncCommand(ri->link->cc, sentinelInfoReplyCallback, ri, "%s", sentinelInstanceMapCommand(ri,"INFO"));
if (retval == C_OK)
{
ri->link->pending_commands++;
}
}
/* Send PING to all the three kinds of instances. */
// 网络传输时网络上耗时 网络延迟 发送 <--> 接收
if ((now - ri->link->last_pong_time) > ping_period && (now - ri->link->last_ping_time) > ping_period/2)
{
sentinelSendPing(ri);
}
/* PUBLISH hello messages to all the three kinds of instances. */
if ((now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD)
{
sentinelSendHello(ri);
}
}
3, sentinel与sentinel服务的通信
sentinel从hello频道获取其他的sentinel的信息的
在sentinelProcessHelloMessage方法中保存其他的sentinel服务的基本信息
中createSentinelRedisInstance创建的新的sentinel服务的信息保存
/**
* 订阅信息的解析的工作
* @param hello 订阅的频道
* @param hello_len 长度
*/
void sentinelProcessHelloMessage(char *hello, int hello_len) {
/* Format is composed of 8 tokens:
* 0=ip,1=port,2=runid,3=current_epoch,4=master_name,
* 5=master_ip,6=master_port,7=master_config_epoch. */
int numtokens, port, removed, master_port;
uint64_t current_epoch, master_config_epoch;
// 127.0.0.1,26380,ba056b537690d37c699f4b482154a684ed6b4d4b,4,mymaster,127.0.0.1,6379,0
char **token = sdssplitlen(hello, hello_len, ",", 1, &numtokens);
sentinelRedisInstance *si, *master;
if (numtokens == 8) {
/* Obtain a reference to the master this hello message is about */
master = sentinelGetMasterByName(token[4]);
//当前sentinel中没有master时服务的信息的 为什么要不处理呢？？？
if (!master)
{
goto cleanup; /* Unknown master, skip the message. */
}
/* First, try to see if we already have this sentinel. */
port = atoi(token[1]);
master_port = atoi(token[6]);
//当前sentinel的服务器没有其他的sentinel服务的信息的需要存储下来的
si = getSentinelRedisInstanceByAddrAndRunID( master->sentinels,token[0],port,token[2]);
current_epoch = strtoull(token[3],NULL,10);
master_config_epoch = strtoull(token[7],NULL,10);
if (!si) {
/* If not, remove all the sentinels that have the same runid
* because there was an address change, and add the same Sentinel
* with the new address back. */
removed = removeMatchingSentinelFromMaster(master,token[2]);
if (removed) {
sentinelEvent(LL_NOTICE,"+sentinel-address-switch",master,
"%@ ip %s port %d for %s", token[0],port,token[2]);
} else {
/* Check if there is another Sentinel with the same address this
* new one is reporting. What we do if this happens is to set its
* port to 0, to signal the address is invalid. We'll update it
* later if we get an HELLO message. */
sentinelRedisInstance *other = getSentinelRedisInstanceByAddrAndRunID(master->sentinels, token[0],port,NULL);
if (other) {
sentinelEvent(LL_NOTICE,"+sentinel-invalid-addr",other,"%@");
other->addr->port = 0; /* It means: invalid address. */
sentinelUpdateSentinelAddressInAllMasters(other);
}
}
/* Add the new sentinel. */
si = createSentinelRedisInstance(token[2],SRI_SENTINEL, token[0],port,master->quorum,master);
if (si) {
if (!removed)
{
sentinelEvent(LL_NOTICE,"+sentinel",si,"%@");
}
/* The runid is NULL after a new instance creation and
* for Sentinels we don't have a later chance to fill it,
* so do it now. */
si->runid = sdsnew(token[2]);
sentinelTryConnectionSharing(si);
if (removed)
{
sentinelUpdateSentinelAddressInAllMasters(si);
}
sentinelFlushConfig();
}
}
/* Update local current_epoch if received current_epoch is greater.*/
if (current_epoch > sentinel.current_epoch) {
sentinel.current_epoch = current_epoch;
sentinelFlushConfig();
sentinelEvent(LL_WARNING,"+new-epoch",master,"%llu",
(unsigned long long) sentinel.current_epoch);
}
/* Update master info if received configuration is newer. */
if (si && master->config_epoch < master_config_epoch) {
master->config_epoch = master_config_epoch;
if (master_port != master->addr->port ||
strcmp(master->addr->ip, token[5]))
{
sentinelAddr *old_addr;
sentinelEvent(LL_WARNING,"+config-update-from",si,"%@");
sentinelEvent(LL_WARNING,"+switch-master", master,"%s %s %d %s %d", master->name, master->addr->ip, master->addr->port, token[5], master_port);
old_addr = dupSentinelAddr(master->addr);
sentinelResetMasterAndChangeAddress(master, token[5], master_port);
sentinelCallClientReconfScript(master, SENTINEL_OBSERVER,"start", old_addr,master->addr);
releaseSentinelAddr(old_addr);
}
}
/* Update the state of the Sentinel. */
if (si) si->last_hello_time = mstime();
}
cleanup:
sdsfreesplitres(token,numtokens);
}
4, sentinel服务控制故障转移的流程
在sentinelCheckObjectivelyDown方法所有sentinel服务的纪元评估该master是否宕机了
把该master的状态修改成SRI_O_DOWN状态
修改master的状态sentinelStartFailoverIfNeeded 方法中sentinelStartFailover修改的成
/* Setup the master state to start a failover. */
void sentinelStartFailover(sentinelRedisInstance *master) {
serverAssert(master->flags & SRI_MASTER);
master->failover_state = SENTINEL_FAILOVER_STATE_WAIT_START;
master->flags |= SRI_FAILOVER_IN_PROGRESS;
// 在这边修改前区纪元增加++1 [非常重要一个步骤 用于分布式同步 sentinel协商重要一个步骤的]
master->failover_epoch = ++sentinel.current_epoch;
sentinelEvent(LL_WARNING,"+new-epoch",master,"%llu", (unsigned long long) sentinel.current_epoch);
sentinelEvent(LL_WARNING,"+try-failover",master,"%@");
master->failover_start_time = mstime()+rand()%SENTINEL_MAX_DESYNC;
master->failover_state_change_time = mstime();
}
在和其他sentinel服务选举中leader来从master服务中评估中slave转换master
在sentinel服务中选举leader服务在每次master服务宕机的都要从新选举出新的leader的sentinel服务
sentinel服务给其他sentinel服务发送 is-master-down-by-addr 命令 分布式一致性Raft算法
选举的规则是先到先是leader 是局部的leader 必须要大于大于全部sentinel服务的数量二分之一加1否则从新在选举的leader服务为止。
在sentinel服务故障转移的状态
switch(ri->failover_state) {
case SENTINEL_FAILOVER_STATE_WAIT_START:
sentinelFailoverWaitStart(ri);
break;
case SENTINEL_FAILOVER_STATE_SELECT_SLAVE:
sentinelFailoverSelectSlave(ri);
break;
case SENTINEL_FAILOVER_STATE_SEND_SLAVEOF_NOONE:
sentinelFailoverSendSlaveOfNoOne(ri);
break;
case SENTINEL_FAILOVER_STATE_WAIT_PROMOTION:
sentinelFailoverWaitPromotion(ri);
break;
case SENTINEL_FAILOVER_STATE_RECONF_SLAVES: //通知其他slave服务同步新的master
sentinelFailoverReconfNextSlave(ri);
break;
}