对ServiceStack.Redis的连接池进行故障转移改造

使用ServiceStack.Redis的连接池在操作多台Redis的时候并不会对出现故障的Redis进行排除切换,这样就会导致应用会还是会分配到故障的Redis服务中导致应用处理错误.这次对ServiceStack.Redis连接池的改造主要实现两个功能:1)对故障的Redis服务在轮循的时候排除,2)定期检测故障的Redis服务,如果服务正常则恢复到轮盾环节中.(ServiceStack.Redis的代码结构还是很不错修改起来也很方便)

增加基于Host的连接池功能

ServiceStack.Redis连接池的连接存储结构相对简单,只是用一些简单的数组进行处理也没有明确按Host划分,所以修改起来比较麻烦.通过查看代码决定在RedisEndPoint添加一系列的功能.包括:连接获取,回收,有效性验测等功能.详细代码如下:

    public class RedisEndPoint : EndPoint
    {
        public RedisEndPoint(string host, int port) : base(host, port)
        {
        }

        public RedisEndPoint(string host, int port, string password) : this(host,port)
        {
            this.Password = password;
        }

        public string Password { get; set; }

        public bool RequiresAuth { get { return !string.IsNullOrEmpty(Password); } }

        private System.Collections.Generic.Stack<RedisClient> mStack = new System.Collections.Generic.Stack<RedisClient>();

        private bool mAvailable = true;

        private int mLastDetectTime = 0;

        private void PingHost(object state)
        {
            try
            {
                RedisClient client = Redis.RedisClientFactory.Instance.CreateRedisClient(Host, Port);
                client.Password = Password;
                client.EndPoint = this;
                client.Ping();
                Push(client);
                mAvailable = true;
                
            }
            catch
            {

            }
            mIsDetecting = false;
        }

        private bool mIsDetecting = false;

        public bool Detect()
        {
            if (!mAvailable)
            {
                if (System.Math.Abs(System.Environment.TickCount - mLastDetectTime) >= 10000)
                {
                    mLastDetectTime = System.Environment.TickCount;
                    if (!mIsDetecting)
                    {
                        mIsDetecting = true;
                        System.Threading.ThreadPool.QueueUserWorkItem(PingHost);
                    }
                    
                   
                }
            }
            return mAvailable;
        }

        public RedisClient Pop()
        {
            lock (mStack)
            {
                if (mStack.Count > 0)
                   return mStack.Pop();
                
            }
            RedisClient client = Redis.RedisClientFactory.Instance.CreateRedisClient(Host, Port);
            client.EndPoint = this;
            client.Password = Password;
            return client;
        }

        public void Push(RedisClient client)
        {
            lock (mStack)
            {
                if (!client.HadExceptions)
                {
                    mStack.Push(client);
                }
                else
                {
                    client.ClientManager = null;
                    client.Dispose();
                    while (mStack.Count > 0)
                    {
                        client = mStack.Pop();
                        client.ClientManager = null;
                        client.Dispose();
                    }
                    mAvailable = false;
                }
            }
        }
    }

比较重要的功能主要是回收和检测,在连接回收的时候判断连接是否存在异常(从代码来看HadExceptions的设置是由SocketError引发的,因此可以判断当这个值为True的时候存在网络异常),如果是则把当前节点标识为不可用,并把池中的所有连接进行清除关闭.检测方法主要是每隔10秒对redis服务进行一个连接和ping操作,如果成功该节点恢复到有效状态. 

修改PooledRedisClientManager 

 为了让新连接池的代码生效,必须修改PooledRedisClientManager几个地方,主要是连接获了和连接回收几个方法的代码.

GetInActiveWriteClient

/// <summary>
        /// Called within a lock
        /// </summary>
        /// <returns></returns>
        private RedisClient GetInActiveWriteClient()
        {
            var desiredIndex = WritePoolIndex % writeClients.Length;
            //this will loop through all hosts in readClients once even though there are 2 for loops
            //both loops are used to try to get the prefered host according to the round robin algorithm
            for (int x = 0; x < ReadWriteHosts.Count; x++)
            {
                var nextHostIndex = (desiredIndex + x) % ReadWriteHosts.Count;
                var nextHost = ReadWriteHosts[nextHostIndex];
                if (nextHost.Detect())
                {
                    RedisClient client = nextHost.Pop();
                    if (client != null)
                    {
                        if (nextHost.RequiresAuth)
                            client.Password = nextHost.Password;
                        client.Id = RedisClientCounter++;
                        client.ClientManager = this;
                        client.NamespacePrefix = NamespacePrefix;
                        client.ConnectionFilter = ConnectionFilter;
                        return client;
                    }
                }
                //for (var i = nextHostIndex; i < writeClients.Length; i += ReadWriteHosts.Count)
                //{                    
                //    if (writeClients[i] != null && !writeClients[i].Active && !writeClients[i].HadExceptions)
                //        return writeClients[i];
                //    else if (writeClients[i] == null || writeClients[i].HadExceptions)
                //    {
                //        if (writeClients[i] != null)
                //            writeClients[i].DisposeConnection();
                //        var client = RedisClientFactory.CreateRedisClient(nextHost.Host, nextHost.Port);

                //        if (nextHost.RequiresAuth)
                //            client.Password = nextHost.Password;

                //        client.Id = RedisClientCounter++;
                //        client.ClientManager = this;
                //        client.NamespacePrefix = NamespacePrefix;
                //        client.ConnectionFilter = ConnectionFilter;
                        
                //        writeClients[i] = client;

                //        return client;
                //    }
                //}
            }
            return null;
        }

把代码改成直接检测当明的Host是否有效,如果是则获取连接并返回,这里只修改的writerclient,类里面还有readclient的方法也相对应用进行修改.

 GetClient方法代码 

        /// <summary>
        /// Returns a Read/Write client (The default) using the hosts defined in ReadWriteHosts
        /// </summary>
        /// <returns></returns>
        public IRedisClient GetClient()
        {
            lock (writeClients)
            {
                AssertValidReadWritePool();

                RedisClient inActiveClient;
                inActiveClient = GetInActiveWriteClient();
                if(inActiveClient == null)
                    throw new TimeoutException(PoolTimeoutError);
                //while ((inActiveClient = GetInActiveWriteClient()) == null)
                //{
                //    if (PoolTimeOut.HasValue)
                //    {
                //        // wait for a connection, cry out if made to wait too long
                //        if (!Monitor.Wait(writeClients, PoolTimeOut.Value))
                //            throw new TimeoutException(PoolTimeoutError);
                //    }
                //    else
                //        Monitor.Wait(writeClients);
                //}

                WritePoolIndex++;
                inActiveClient.Active = true;

                if (this.ConnectTimeout != null)
                {
                    inActiveClient.ConnectTimeout = this.ConnectTimeout.Value;
                }

                if( this.SocketSendTimeout.HasValue )
                {
                    inActiveClient.SendTimeout = this.SocketSendTimeout.Value;
                }
                if( this.SocketReceiveTimeout.HasValue )
                {
                    inActiveClient.ReceiveTimeout = this.SocketReceiveTimeout.Value;
                }

                inActiveClient.NamespacePrefix = NamespacePrefix;

                //Reset database to default if changed
                if (inActiveClient.Db != Db)
                {
                    inActiveClient.ChangeDb(Db);
                }

                return inActiveClient;
            }
        }

DisposeClient方法代码

        public void DisposeClient(RedisNativeClient client)
        {
            if (client.EndPoint != null)
            {
                client.EndPoint.Push((RedisClient)client);
                return;
            }
            //lock (readClients)
            //{
            //    for (var i = 0; i < readClients.Length; i++)
            //    {
            //        var readClient = readClients[i];
            //        if (client != readClient) continue;
            //        client.Active = false;
            //        Monitor.PulseAll(readClients);
            //        return;
            //    }
            //}

通过以上简单的代码修改后ServiceStack.Redis的连接池就具备了故意迁移和恢复的功能:)