Redis BIgKey自动分析处理

    使用开源rdb工具与Lepus进行REDIS RDB的数据分析,相信使用REDIS,对CACHE的重要性不言而喻,尽可做到事前-预判,坚决不做背锅侠........

    生成环境需要分析的RDB会很多,如一个FE展示着多个DASHBOARD,一个DASHBOARD下有一个或多个GROUP,监控的REDIS主要从Lepus获取(可根据实际是否使用),如何避免手动逐个RDB进行分析,进行重复操作....................

    实现过程:

         1、结合LEPUS 查找存在的REDIS

             1、主机信息:存储ROOT密码用来PARAMIKO登陆

             2、LEPUS定义监控信息

         2、从FE获取记录需要分析的RDB

         3、执行分析邮件告警

      ###实践过程需要解决python一些依赖包:MySQL Redis RDB  

pip install python-lzf

定义表:

 int(11) NOT NULL AUTO_INCREMENT,
  `idc` tinyint(1) DEFAULT '1',
  `productName` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '业务线名称',
  `ip` varchar(16) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'REDIS地址',
  `port` int(11) DEFAULT NULL COMMENT 'REDIS端口',
  `bgsave` tinyint(1) DEFAULT '0' COMMENT '执行状态',
  `rdb_last_save_time_begin` int(11) DEFAULT '1461645350' COMMENT '第一次执行前时间',
  `rdb_last_save_time_over` int(11) DEFAULT '1461645350' COMMENT '执行BGSAVE后时间',
  `rdb_last_bgsave_status` varchar(10) COLLATE utf8mb4_unicode_ci DEFAULT 'ok',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT "业务线定义分析的RDB"

 

CREATE TABLE `mysql_ssh_pass` (
  `id` int(32) NOT NULL AUTO_INCREMENT,
  `idc` int(16) DEFAULT NULL COMMENT '所述IDC编号',
  `ip` varchar(15) CHARACTER SET utf8 DEFAULT NULL COMMENT '数据库ip',
  `os` varchar(30) CHARACTER SET utf8 DEFAULT NULL COMMENT '服务器系统版本',
  `username` varchar(20) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh用户名',
  `password` varchar(128) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh密码',
  `comment` varchar(500) CHARACTER SET utf8 NOT NULL DEFAULT '' COMMENT '备注',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uniqa` (`ip`)
) ENGINE=InnoDB AUTO_INCREMENT=307 DEFAULT CHARSET=utf8mb4 COMMENT "定义系统登陆信息"

###可有可无,主要使用LEPUS 可以直接使用

CREATE TABLE `db_servers_redis` (
  `id` int(10) NOT NULL AUTO_INCREMENT,
  `host` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
  `port` varchar(10) COLLATE utf8_unicode_ci NOT NULL,
  `password` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
  `tags` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
  `monitor` tinyint(2) DEFAULT '1',
  `send_mail` tinyint(2) DEFAULT '1',
  `send_mail_to_list` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
  `send_wx` tinyint(4) DEFAULT '1',
  `send_wx_to_list` tinyint(255) DEFAULT NULL,
  `send_sms` tinyint(2) DEFAULT '0',
  `send_sms_to_list` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `alarm_connected_clients` int(2) NOT NULL DEFAULT '1',
  `alarm_command_processed` int(2) NOT NULL DEFAULT '1',
  `alarm_blocked_clients` int(2) NOT NULL DEFAULT '1',
  `threshold_warning_connected_clients` int(4) NOT NULL DEFAULT '1000',
  `threshold_warning_command_processed` int(4) NOT NULL DEFAULT '12000',
  `threshold_warning_blocked_clients` int(4) NOT NULL DEFAULT '50',
  `threshold_critical_connected_clients` int(4) NOT NULL DEFAULT '3000',
  `threshold_critical_command_processed` int(4) NOT NULL DEFAULT '15000',
  `threshold_critical_blocked_clients` int(4) NOT NULL DEFAULT '150',
  `is_delete` tinyint(1) NOT NULL DEFAULT '0',
  `display_order` smallint(4) NOT NULL DEFAULT '0',
  `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`),
  KEY `idx_host` (`host`)
) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci

###邮件模板SendMail.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
#from html import HTML
from email.utils import formataddr
reload(sys) 
sys.setdefaultencoding('utf-8')
def sendSmail(subject,msg,fromemail,emailpasswd,toemail):
	user=fromemail
        pwd=emailpasswd
        to=toemail
        nowtime=time.strftime('%Y-%m-%d %H:%M:%S')
        msg=MIMEText(msg,'html','utf-8')
        if not isinstance(subject,unicode):
            subject = unicode(subject)
        msg["Accept-Language"]="zh-CN"
        msg["Accept-Charset"]="ISO-8859-1,utf-8"
        msg["Subject"]=subject
        msg["From"]=user
        msg["To"]=",".join(to)
        try:
                s=smtplib.SMTP('mail.hualala.com',587)
                s.login(user,pwd)
                s.sendmail(fromemail,to,msg.as_string())
                s.quit()
                print "[%s]INFO:Email send Success!"% nowtime
        except smtplib.SMTPException,e:
                print "[%s]ERROR:Email send Faild,%s"%(nowtime,e)

##告警模板alarm_sendmail.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
from email.utils import formataddr
import SendMail as sendMail
reload(sys) 
sys.setdefaultencoding('utf-8')
def sendSlogLog(host,port,data,productName,sendList):
	bb=""
	userlist1='''
        

Codis BigKeys Analysis:'''+str(host)+"_"+str(port)+" "+"【"+productName+"】"+" "+'''

''' with open(data,'r') as f: for line in f.readlines(): data=line.strip().split(',') aa=''''''%(str(data[0]),str(data[1]),str(data[2]),str(data[3]),str(data[4]),str(data[5]),str(data[6]),str(data[7])) bb=bb+aa+"\n" if bb: cc=userlist1+bb+"
Database Type Key Size_in_bytes Encoding Num_elements Len_largest_element Expiry
%s %s %s %s %s %s %s %s
" subject="Redis BigKey Analysis Report" fromemail='邮箱地址' emailpasswd='邮箱密码' global to_addrs if sendList.strip()=='': sendList="[email protected];[email protected]" else: sendList=sendList toemail=sendList to_addrs=toemail.split(';') sendMail.sendSmail(subject,cc,fromemail,emailpasswd,to_addrs)

##执行模板:rbd-exe.py

#!/usr/bin/evn python
#-*- coding: utf-8 -*-  
import sys,os,commands,subprocess
import paramiko,socket,time
import alarm_sendmail as al
import base64
reload(sys)
sys.setdefaultencoding('utf-8')
import redis,MySQLdb
def ConnRedis(rdb):
    backup=rdb
    host=rdb['ip']
    port=rdb['port']
    pool=redis.ConnectionPool(host=host,port=port,db=0,decode_responses=True)
    r=redis.Redis(connection_pool=pool)
    cur=conn.cursor()
    if backup['CurrentDate'] !=backup['rdb_last_save_time_over'] and backup['bgsave']==0:
        print "为执行bgsave.......",backup['CurrentDate'],backup['rdb_last_save_time_over']
        try:
	    beginStart=int(time.time())
	    up="update db_manager.tbl_bigdate_record set bgsave=1 where ip="+"'"+str(host)+"'"+"and port="+str(port)
	    print "执行RDB备份中.........",beginStart
	    cur.execute(up)
            conn.commit()
            r.bgsave()
	    time.sleep(30)
	    info=r.info()
	    sdirName=r.config_get()['dir']
	    dumpName=r.config_get()['dbfilename']
	    rdb_back_status=''
	    result=''
	    if info['rdb_last_bgsave_status']=="ok":
                rdb_back_status='ok'
		result="执行RDB备份成功,完成时间:%s....."%info['rdb_last_save_time']
	    else:
		rdb_back_status='err'
		result="执行备份失败.....Codis:%s:%s"%(host,port)
	    print result
	    endup="update db_manager.tbl_bigdate_record set bgsave=0,rdb_last_save_time_over="+str(info['rdb_last_save_time'])+",rdb_last_bgsave_status="+"'"+str(rdb_back_status)+"'"+",rdb_last_save_time_begin="+str(beginStart)+" where ip="+"'"+str(host)+"'"+"and port="+str(port)
	    cur.execute(endup)
	    conn.commit()
	    return rdb_back_status,sdirName,dumpName
        except Exception,e:
            print e
    elif backup['CurrentDate']==backup['rdb_last_save_time_over'] and backup['bgsave']==0 and backup['rdb_last_bgsave_status']=='err':
        print "RDB Backup 异常..........Codis:%s:%s"%(host,port)
    else:
	rdb_back_status=3
	print "RDB Backup 可以使用上次备份分析......Codis:%s:%s"%(host,port)
	return rdb_back_status,1

def ConnDB(host,port,username,password,idc):
    #backup={}
    conn=MySQLdb.connect(host=host,port=port,user=username,passwd=password,charset='utf8')
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    #sql="select productName,ip,port,rdbconf,bgsave,date(from_unixtime(rdb_last_save_time_begin))as rdb_last_save_time_begin,date(from_unixtime(rdb_last_save_time_over))as rdb_last_save_time_over,rdb_last_bgsave_status,date(now())as CurrentDate from db_manager.tbl_bigdate_record"
    sql="SELECT cmd.productName as productName,cmd.ip as ip,cmd.port as port,cmd.bgsave as bgsave,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_begin))AS rdb_last_save_time_begin,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_over))AS rdb_last_save_time_over,cmd.rdb_last_bgsave_status,DATE(NOW())AS CurrentDate,tmp.send_mail_to_list,pwd.password AS upassword FROM db_manager.tbl_bigdate_record cmd LEFT JOIN lepus.db_servers_redis tmp ON cmd.ip=tmp.host AND cmd.port=tmp.port LEFT JOIN dbinfo.mysql_ssh_pass pwd  ON pwd.ip=cmd.ip WHERE tmp.host IS NOT NULL and cmd.idc={idc}".format(idc=str(idc)) 
    cur.execute(sql)
    backup=cur.fetchall()
#    for i in data:
#        for k,v in i.items():
#           backup[k]=v

    return backup,conn
def LoadSoft(logdir,rdb):
    ##下载包保存文件存储空间检查...................解压存储路径检擦
    Rdbcodis=ConnRedis(rdb)
    if not Rdbcodis:
	print "未获取到最新得rdb文件........"
    elif  Rdbcodis[0]=="err":
	print "RDB异常......"
    elif Rdbcodis[0]==3:
        print "已分析过RDB......."
    else:
	password_tmp=rdb['upassword']
        upassword=base64.decodestring(password_tmp)
        remote_file=Rdbcodis[1]
	ip=rdb['ip']
        productName=rdb['productName']
        local_file_tmp=logdir+"/"+str(productName)
        print remote_file,local_file_tmp
        if not os.path.exists(local_file_tmp):
	    os.system(r'mkdir -p {}'.format(local_file_tmp))
        remote_file=remote_file+"/"+str(Rdbcodis[2])
        local_file=local_file_tmp+"/"+str(Rdbcodis[2])
        ssh=paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(ip,22,'root',upassword)
        a=ssh.exec_command('data')
        stdin,stdout,stdeer=a
        print stdout.read()
        sftp=paramiko.SFTPClient.from_transport(ssh.get_transport())
        time1=time.time()
        sftp=ssh.open_sftp()
        sftp.get(remote_file,local_file)
        gdown=True
    
        print "下载完成%s,Codis:%s:%s"%(str(time.time()-time1),ip,rdb['port'])
	return local_file
def RdbAnalyse(logdir,rdb):
    newdstDir=LoadSoft(logdir,rdb)
    if not newdstDir:
        print "无任何操作........"
    else:
	beginStart=int(time.time())
        print "执行RDB分析.........",beginStart
        dstDir=newdstDir
	sendList=rdb['send_mail_to_list']
	productName=rdb['productName']
	host=rdb['ip']
	port=rdb['port']
        cmd_exe="/usr/local/bin/rdb -c memory -l 100 "+dstDir
        dstDumpDir=os.path.dirname(dstDir)+"/"+productName+".log"
        log_file=open(dstDumpDir,"w")
        stdout_rdb=sys.stdout
	sys.stdout=log_file
	with os.popen(cmd_exe)as c:
	    rdb_log=c.read()
	    print rdb_log
	log_file.close()
	sys.stdout=stdout_rdb
	cmddel="sed -i 1d"+" "+dstDumpDir
	os.system(cmddel)
	cmdr="sed -i '/^$/d'"+" "+dstDumpDir
	os.system(cmdr)
	endStart=int(time.time())
        print "执行RDB分析完成,共计耗时:%s........."%(endStart-beginStart)
	al.sendSlogLog(host,port,dstDumpDir,productName,sendList)
if __name__=="__main__":
    idc=sys.argv[1]
    logdir="/home/rdb-analyse/analyse"
    sqlbackup_tmp=ConnDB('1.1.1.1',123,'tet','abc',idc)
    conn=sqlbackup_tmp[1]
    for rdb in sqlbackup_tmp[0]:
        RdbAnalyse(logdir,rdb)
 #       else:
  #          print "无匹配项"

###最终执行:python rdb-exe.py number

Redis BIgKey自动分析处理_第1张图片

你可能感兴趣的:(REDIS)