SQL Server Alwayson架构下 服务器 各虚拟IP漂移监控告警的功能实现 -1（服务器视角）

1.需求概括

我们知道，在SQL Server Alwayson 架构中，有多种虚拟IP，例如 WindowsCluster IP，ListenIP，角色高可用性IP（类似于侦听IP）。在某些条件下，例如系统故障，会触发虚拟IP的漂移，如何高效率、低延迟、更好地监控IP漂移情况，是我们DB的一个重要工作。

下面是我们的一个通过SQL Server 存储过程来实现的方案。

2.基本原理 

 周期性收集当前Server 上相应的IP地址，并与上个周期收集的结果比较判断，看那些IP发生了漂移变化。其主要流程图如下;

 

 

3.代码实现

表 DBA_ServerIPDataBase_OverCheck，主要存储 当前 （本收集周期） Server的信息（主要是IP信息、ServerName信息等），其创建脚本如下;

/****** Object:  Table [dbo].[DBA_ServerIPDataBase_OverCheck]    Script Date: 2019/6/27 16:01:27 ******/
SET ANSI_NULLS ON
GO

SET QUOTED_IDENTIFIER ON
GO

SET ANSI_PADDING ON
GO

CREATE TABLE [dbo].[DBA_ServerIPDataBase_OverCheck](
    [LocalServerIP] [varchar](20) NULL,
    [ClusterName] [varchar](50) NULL,
    [ServerIP] [varchar](20) NULL,
    [ServerName] [varchar](100) NULL,
    [ServerFullName] [varchar](100) NULL,
    [ServerIPType] [varchar](20) NULL,
    [DataBaseName] [varchar](300) NULL,
    [DisabledFlag] [varchar](1) NULL,
    [CreateTime] [datetime] NULL,
    [CreateBy] [varchar](50) NULL,
    [ModifyTime] [datetime] NULL,
    [ModifyBy] [varchar](50) NULL
) ON [PRIMARY]

GO

SET ANSI_PADDING OFF
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Windows集群名称' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'ClusterName'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'IP地址' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'ServerIP'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'计算机对象名称' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'ServerName'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'计算机对象全称' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'ServerFullName'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'计算机对象全称' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'ServerIPType'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'0实时有效，1第一次失效，2第二次失效，3第三次失效，4第四次失效，5第五次失效，彻底删除' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'DisabledFlag'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'创建时间' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'CreateTime'
GO

EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'创建人' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'DBA_ServerIPDataBase_OverCheck', @level2type=N'COLUMN',@level2name=N'CreateBy'
GO

 

表 DBA_ServerIPDataBase_OverCheckOriginOrigin，主要存储 以前 （前一个收集周期） Server的信息，是用来比较变化的基准表，其表结构 与表DBA_ServerIPDataBase_OverCheck相同，创建脚本如下：

 

/****** Object:  Table [dbo].[DBA_ServerIPDataBase_OverCheckOriginOrigin]    Script Date: 2019/6/27 16:56:28 ******/
SET ANSI_NULLS ON
GO

SET QUOTED_IDENTIFIER ON
GO

SET ANSI_PADDING ON
GO

CREATE TABLE [dbo].[DBA_ServerIPDataBase_OverCheckOriginOrigin](
    [LocalServerIP] [varchar](20) NULL,
    [ClusterName] [varchar](50) NULL,
    [ServerIP] [varchar](20) NULL,
    [ServerName] [varchar](100) NULL,
    [ServerFullName] [varchar](100) NULL,
    [ServerIPType] [varchar](20) NULL,
    [DataBaseName] [varchar](300) NULL,
    [DisabledFlag] [varchar](1) NULL,
    [CreateTime] [datetime] NULL,
    [CreateBy] [varchar](50) NULL,
    [ModifyTime] [datetime] NULL,
    [ModifyBy] [varchar](50) NULL
) ON [PRIMARY]

GO

SET ANSI_PADDING OFF
GO

 

具体的存储过程 USP_DBA_ServerIPDataBase_OverCheck，用来实现 收集、比较、告警等功能，代码实现如下：

 

/****** Object:  StoredProcedure [dbo].[USP_DBA_ServerIPDataBase_OverCheck]    Script Date: 2019/6/27 15:26:06 ******/
SET ANSI_NULLS ON
GO

SET QUOTED_IDENTIFIER ON
GO


CREATE PROCEDURE [dbo].[USP_DBA_ServerIPDataBase_OverCheck]
AS
BEGIN 

DECLARE @ipline VARCHAR(200) 
        DECLARE @ip VARCHAR(20)
        DECLARE @pingname VARCHAR(40)
        DECLARE @ServerName VARCHAR(100)
        DECLARE @ServerFullName VARCHAR(100)

        Declare @CurrDateTime nvarchar(20)
        Declare @PreDiffDateTime nvarchar(20) =''

        Set @CurrDateTime=CONVERT(VARCHAR(19),GETDATE(),120) 
 
        SET NOCOUNT ON

        Declare @ServerIP VARCHAR(20)
        SET @ServerName=@@SERVERNAME
        SET @ServerIP = CAST(CONNECTIONPROPERTY('local_net_address') AS varchar(20)) 

        ----Set @ServerIP ='XXX.XXX.XXX.XXX'   --如果不准确的话，请手动定义
            
        DELETE FROM DBA_ServerIPDataBase_OverCheck 
        WHERE CreateTime< CONVERT(VARCHAR(19),DATEADD( HH,-10,GETDATE()),120) 

        SET @ip = NULL 
        IF OBJECT_ID('tempdb..#tempserverip') IS NOT NULL
            DROP TABLE #tempserverip 
        CREATE TABLE #tempserverip ( ipline VARCHAR(200) ) 
        INSERT  #tempserverip
                EXEC master..xp_cmdshell 'ipconfig /all ' 
        
        INSERT  INTO DBA_ServerIPDataBase_OverCheck
                ( LocalServerIP,ServerIP,DisabledFlag,CreateTime,CreateBy
                )
                SELECT @ServerIP, p.ServerIP,'0'AS DisabledFlag,GETDATE() AS CreateTime ,@@SERVERNAME AS CreateBy
                FROM    dbo.DBA_ServerIPDataBase_OverCheck i
                        RIGHT JOIN ( SELECT RTRIM(LTRIM(REPLACE(SUBSTRING(ipline,
                                                              CHARINDEX(':',
                                                              ipline) + 1, 20),
                                                              '(首选)', ''))) AS 'ServerIP'
                                     FROM   #tempserverip
                                     WHERE  UPPER(ipline) LIKE '%IPv4 地址%'--这里需要注意一下，系统不同这里的匹配值就不同 
                                            AND UPPER(ipline) NOT LIKE '%192.168.%'
                                            AND UPPER(ipline) NOT LIKE '%169.254.%'
                                   ) p ON i.ServerIP = p.ServerIP
                WHERE   i.ServerIP IS NULL  --只关注漂来飘往数据

    --0002 -20180530 针对20180530持续告警问题，发现告警时间超过预期，进行优化。聚焦点再表DBA_ServerIPDataBase_OverCheck中的CreateTime栏位，精准更新  begin
    update i set i.CreateTime=getdate()
    FROM    dbo.DBA_ServerIPDataBase_OverCheck i
                        RIGHT JOIN ( SELECT RTRIM(LTRIM(REPLACE(SUBSTRING(ipline,
                                                              CHARINDEX(':',
                                                              ipline) + 1, 20),
                                                              '(首选)', ''))) AS 'ServerIP'
                                     FROM   #tempserverip
                                     WHERE  UPPER(ipline) LIKE '%IPv4 地址%'--这里需要注意一下，系统不同这里的匹配值就不同 
                                            AND UPPER(ipline) NOT LIKE '%192.168.%'
                                            AND UPPER(ipline) NOT LIKE '%169.254.%'
                                   ) p ON i.ServerIP = p.ServerIP

    ------
        DECLARE IP CURSOR
        FOR
            SELECT  ServerIP
            FROM    dbo.DBA_ServerIPDataBase_OverCheck WHERE DisabledFlag IS NOT NULL AND DisabledFlag=0
        OPEN IP
        FETCH NEXT FROM IP INTO @ip
        WHILE @@FETCH_STATUS = 0
            BEGIN
                --SET @pingname = 'ping -a ' + @ip
                SET @pingname = 'ping -a ' + @ip + ' -n 1 -l 10'

                TRUNCATE TABLE #tempserverip
                INSERT  #tempserverip
                        EXEC master..xp_cmdshell @pingname

                SELECT  @ServerName = REPLACE(RTRIM(LTRIM(SUBSTRING(ipline, 8,
                                                              CHARINDEX('[',
                                                              ipline) - 8))),
                                              '.XXXXXX.com', '') ,-----加域的电脑，计算机名字可能带有域名，请根据实际情况替换
                        @ServerFullName = RTRIM(LTRIM(SUBSTRING(ipline, 8,
                                                              CHARINDEX('[',
                                                              ipline) - 8)))
                FROM    #tempserverip
                WHERE   ipline LIKE '%正在 Ping%'

                UPDATE  dbo.DBA_ServerIPDataBase_OverCheck
                SET     ServerName = @ServerName ,
                        ServerFullName = @ServerFullName
                WHERE   ServerIP = @ip

                FETCH NEXT FROM IP INTO @ip
            END
        CLOSE IP 
        DEALLOCATE IP

        UPDATE  dbo.DBA_ServerIPDataBase_OverCheck
        SET     ServerIPType = 'Localhost'
        WHERE   ServerName = @@SERVERNAME


        if OBJECT_ID('sys.availability_group_listener_ip_addresses') IS NOT NULL 
        begin
          update a set a.ServerIPType='ListenIP'
          from DBA_ServerIPDataBase_OverCheck a inner join sys.availability_group_listener_ip_addresses b
          on a.ServerIP=b.ip_address
        end 

        IF OBJECT_ID('sys.dm_hadr_cluster') IS NOT NULL
            BEGIN
                UPDATE dbo.DBA_ServerIPDataBase_OverCheck SET ClusterName=(SELECT cluster_name FROM sys.dm_hadr_cluster)
                UPDATE dbo.DBA_ServerIPDataBase_OverCheck SET ServerIPType='WindowsCluster' WHERE ServerName=(SELECT cluster_name FROM sys.dm_hadr_cluster)
            END
        IF OBJECT_ID('sys.dm_hadr_cluster') IS NULL
            BEGIN
                UPDATE dbo.DBA_ServerIPDataBase_OverCheck SET ClusterName='Not Cluster'
            END
--0002 -20180530  针对20180530持续告警问题，发现告警时间超过预期，进行优化。聚焦点再表DBA_ServerIPDataBase_OverCheck中的CreateTime栏位，精准更新  begin
        UPDATE DBA_ServerIPDataBase_OverCheck SET  DataBaseName=STUFF(
            (SELECT ',' + name FROM sys.databases  
                               WHERE name not in ('master'
                                 ,'tempdb','model','msdb','ReportServer','ReportServerTempDB','distribution')
                               for xml path('')   ),
                            1,1,'')
-----002 end 
        --the mail alarm 
           declare @SQL as varchar(200)
        declare @Subject as varchar(200)=N'DB SERVER IP 有漂移,请检查确认！'
        declare @Body as nvarchar(max)=''

        select @PreDiffDateTime= CreateTime  from DBA_ServerIPDataBase_OverCheckOriginOrigin
        order by CreateTime
        
        SELECT  TOP 0 A.*  into #temp_DBA_ServerIPDataBase_OverCheck_diff FROM DBA_ServerIPDataBase_OverCheck A INNER JOIN DBA_ServerIPDataBase_OverCheckOriginOrigin  B
         ON A.ServerIP =B.ServerIP  

        IF EXISTS(SELECT * FROM DBA_ServerIPDataBase_OverCheck A LEFT JOIN DBA_ServerIPDataBase_OverCheckOriginOrigin  B ON A.ServerIP =B.ServerIP  WHERE B.ServerIP IS NULL)
        BEGIN

           INSERT  INTO  #temp_DBA_ServerIPDataBase_OverCheck_diff
           SELECT A.*  FROM DBA_ServerIPDataBase_OverCheck A LEFT JOIN DBA_ServerIPDataBase_OverCheckOriginOrigin  B ON A.ServerIP =B.ServerIP  WHERE B.ServerIP IS NULL

            if exists(    select * from #temp_DBA_ServerIPDataBase_OverCheck_diff  ) 
            begin
            set @Body= N'' 
                    + N''
                    + N'
' 
            SET @Body=@Body+'Dear All,

     此List是监控到过去10 MIn Server IP 异常情况,'+@ServerIP +'服务器有新IP创建声明（新增）， 请及时Check。具体数据如下：;

'SET@Body=@Body+''SELECT@Body=@Body+''from #temp_DBA_ServerIPDataBase_OverCheck_diff     

             SET@Body=@Body+'
 
             
ClusterName
ServerIP
ServerName
ServerIPType
DatabaseName
DisabledFlag
当前时间
差异采样时间
    
             
'+ClusterName+'
'+ServerIP+'
'+ServerName+'
'+ServerIPType+'
'+DatabaseName+'
'+ DisabledFlag+'
'+@CurrDateTime+'
'+ @PreDiffDateTime+'
                 


DBA
Best wishes'
            end 
       END


        IF EXISTS(SELECT * FROM DBA_ServerIPDataBase_OverCheck A RIGHT JOIN DBA_ServerIPDataBase_OverCheckOriginOrigin  B ON A.ServerIP =B.ServerIP  WHERE A.ServerIP IS NULL)
        BEGIN

           DELETE FROM #temp_DBA_ServerIPDataBase_OverCheck_diff
           INSERT  INTO  #temp_DBA_ServerIPDataBase_OverCheck_diff
           SELECT A.*  FROM DBA_ServerIPDataBase_OverCheck A RIGHT JOIN DBA_ServerIPDataBase_OverCheckOriginOrigin  B ON A.ServerIP =B.ServerIP  WHERE A.ServerIP IS NULL

            if exists(    select * from #temp_DBA_ServerIPDataBase_OverCheck_diff  ) 
            begin
            set @Body= N'' 
                    + N''
                    + N'
' 
            SET @Body=@Body+'Dear All,

     此List是监控到过去10 MIn Server IP 异常情况,'+@ServerIP +'服务器有IP漂移（消减）， 请及时Check。具体数据如下：;

'SET@Body=@Body+''SELECT@Body=@Body+''from #temp_DBA_ServerIPDataBase_OverCheck_diff     

             SET@Body=@Body+'
 
             
ClusterName
ServerIP
ServerName
ServerIPType
DatabaseName
DisabledFlag
当前时间
差异采样时间
    
             
'+ClusterName+'
'+ServerIP+'
'+ServerName+'
'+ServerIPType+'
'+DatabaseName+'
'+ DisabledFlag+'
'+@CurrDateTime+'
'+ @PreDiffDateTime+'
                 


DBA
Best wishes'
            end 

       END  
            SET @BODY=REPLACE(@BODY,'''','')
            IF REPLACE(@BODY,' ','')<>''
                BEGIN

                    Declare @AllEmailToAddress varchar(3000)=''
                    Declare @AllEmailCcAddress varchar(3000)=''
                    Select @AllEmailToAddress='hanmeimei;xiaoming;lilei'

                    Select @AllEmailCcAddress='laoban'


                    exec msdb..sp_send_dbmail @profile_name =  'AutoMail'               -- profile 名称，请检查此参数，根据实际情况进行替换 
                     ,@recipients   =  @AllEmailToAddress         -- 收件人邮箱 
                     ,@copy_recipients=@AllEmailCcAddress
                     ,@subject      =  @Subject -- 邮件标题 
                     ,@body         =  @BODY            -- 邮件内容 
                     ,@body_format  =  'HTML'                      -- 邮件格式 
                     ,@file_attachments=''
                     ,@importance = 'HIGH' -- varchar(10) 告警级别
                END


        ------------------新增立即插入-----------

        insert into [dbo].[DBA_ServerIPDataBase_OverCheckOriginOrigin]
        select a.* from  DBA_ServerIPDataBase_OverCheck a left join DBA_ServerIPDataBase_OverCheckOriginOrigin b
        on a.ServerIP=b.ServerIP
        where b.ServerIP is null

        ---漂移后，指定时间段后直接删除过时数据,暂定八个小时。【即如果有漂移（减少），减少的IP信息，则在指定时间后，删除。】

        delete b
        from  DBA_ServerIPDataBase_OverCheck a right join DBA_ServerIPDataBase_OverCheckOriginOrigin b
        on a.ServerIP=b.ServerIP
        where a.ServerIP is null
        and b.CreateTime< CONVERT(VARCHAR(19),DATEADD( HH,-3,GETDATE()),120)

       -----将数据插入到远程Server DB中，远程Server有一个SP过程，用来判断漂移前/后DB是否有变化。远程的SP主要是依据 LocalServerIP 和 ServerIP 对应关系变化情况来判断。这段代码省略，下次再描述

        SET NOCOUNT OFF 

End


GO

 

 4.功能实现

例如当 服务器有新IP创建声明（新增）时，其发出的告警邮件如下：

 

 

 

本文版权归作者所有，未经作者同意不得转载,谢谢配合！！！

本文版权归作者所有，未经作者同意不得转载,谢谢配合！！！

本文版权归作者所有，未经作者同意不得转载,谢谢配合！！！