Redis BIgKey自动分析处理
使用开源rdb工具与Lepus进行REDIS RDB的数据分析，相信使用REDIS，对CACHE的重要性不言而喻，尽可做到事前-预判，坚决不做背锅侠........
生成环境需要分析的RDB会很多，如一个FE展示着多个DASHBOARD，一个DASHBOARD下有一个或多个GROUP，监控的REDIS主要从Lepus获取(可根据实际是否使用)，如何避免手动逐个RDB进行分析，进行重复操作....................
实现过程:
1、结合LEPUS 查找存在的REDIS
1、主机信息：存储ROOT密码用来PARAMIKO登陆
2、LEPUS定义监控信息
2、从FE获取记录需要分析的RDB
3、执行分析邮件告警
###实践过程需要解决python一些依赖包:MySQL Redis RDB
pip install python-lzf
定义表:
int(11) NOT NULL AUTO_INCREMENT,
`idc` tinyint(1) DEFAULT '1',
`productName` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '业务线名称',
`ip` varchar(16) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'REDIS地址',
`port` int(11) DEFAULT NULL COMMENT 'REDIS端口',
`bgsave` tinyint(1) DEFAULT '0' COMMENT '执行状态',
`rdb_last_save_time_begin` int(11) DEFAULT '1461645350' COMMENT '第一次执行前时间',
`rdb_last_save_time_over` int(11) DEFAULT '1461645350' COMMENT '执行BGSAVE后时间',
`rdb_last_bgsave_status` varchar(10) COLLATE utf8mb4_unicode_ci DEFAULT 'ok',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT "业务线定义分析的RDB"
CREATE TABLE `mysql_ssh_pass` (
`id` int(32) NOT NULL AUTO_INCREMENT,
`idc` int(16) DEFAULT NULL COMMENT '所述IDC编号',
`ip` varchar(15) CHARACTER SET utf8 DEFAULT NULL COMMENT '数据库ip',
`os` varchar(30) CHARACTER SET utf8 DEFAULT NULL COMMENT '服务器系统版本',
`username` varchar(20) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh用户名',
`password` varchar(128) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh密码',
`comment` varchar(500) CHARACTER SET utf8 NOT NULL DEFAULT '' COMMENT '备注',
PRIMARY KEY (`id`),
UNIQUE KEY `uniqa` (`ip`)
) ENGINE=InnoDB AUTO_INCREMENT=307 DEFAULT CHARSET=utf8mb4 COMMENT "定义系统登陆信息"
###可有可无，主要使用LEPUS 可以直接使用
CREATE TABLE `db_servers_redis` (
`id` int(10) NOT NULL AUTO_INCREMENT,
`host` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
`port` varchar(10) COLLATE utf8_unicode_ci NOT NULL,
`password` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`tags` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
`monitor` tinyint(2) DEFAULT '1',
`send_mail` tinyint(2) DEFAULT '1',
`send_mail_to_list` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
`send_wx` tinyint(4) DEFAULT '1',
`send_wx_to_list` tinyint(255) DEFAULT NULL,
`send_sms` tinyint(2) DEFAULT '0',
`send_sms_to_list` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
`alarm_connected_clients` int(2) NOT NULL DEFAULT '1',
`alarm_command_processed` int(2) NOT NULL DEFAULT '1',
`alarm_blocked_clients` int(2) NOT NULL DEFAULT '1',
`threshold_warning_connected_clients` int(4) NOT NULL DEFAULT '1000',
`threshold_warning_command_processed` int(4) NOT NULL DEFAULT '12000',
`threshold_warning_blocked_clients` int(4) NOT NULL DEFAULT '50',
`threshold_critical_connected_clients` int(4) NOT NULL DEFAULT '3000',
`threshold_critical_command_processed` int(4) NOT NULL DEFAULT '15000',
`threshold_critical_blocked_clients` int(4) NOT NULL DEFAULT '150',
`is_delete` tinyint(1) NOT NULL DEFAULT '0',
`display_order` smallint(4) NOT NULL DEFAULT '0',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_host` (`host`)
) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
###邮件模板SendMail.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
#from html import HTML
from email.utils import formataddr
reload(sys)
sys.setdefaultencoding('utf-8')
def sendSmail(subject,msg,fromemail,emailpasswd,toemail):
user=fromemail
pwd=emailpasswd
to=toemail
nowtime=time.strftime('%Y-%m-%d %H:%M:%S')
msg=MIMEText(msg,'html','utf-8')
if not isinstance(subject,unicode):
subject = unicode(subject)
msg["Accept-Language"]="zh-CN"
msg["Accept-Charset"]="ISO-8859-1,utf-8"
msg["Subject"]=subject
msg["From"]=user
msg["To"]=",".join(to)
try:
s=smtplib.SMTP('mail.hualala.com',587)
s.login(user,pwd)
s.sendmail(fromemail,to,msg.as_string())
s.quit()
print "[%s]INFO:Email send Success!"% nowtime
except smtplib.SMTPException,e:
print "[%s]ERROR:Email send Faild,%s"%(nowtime,e)
##告警模板alarm_sendmail.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
from email.utils import formataddr
import SendMail as sendMail
reload(sys)
sys.setdefaultencoding('utf-8')
def sendSlogLog(host,port,data,productName,sendList):
bb=""
userlist1='''
Codis BigKeys Analysis:'''+str(host)+"_"+str(port)+" "+"【"+productName+"】"+" "+'''
Database
Type
Key
Size_in_bytes
Encoding
Num_elements
Len_largest_element
Expiry
'''
with open(data,'r') as f:
for line in f.readlines():
data=line.strip().split(',')
aa='''
%s
%s
%s
%s
%s
%s
%s
%s
'''%(str(data[0]),str(data[1]),str(data[2]),str(data[3]),str(data[4]),str(data[5]),str(data[6]),str(data[7]))
bb=bb+aa+"\n"
if bb:
cc=userlist1+bb+"
"
subject="Redis BigKey Analysis Report"
fromemail='邮箱地址'
emailpasswd='邮箱密码'
global to_addrs
if sendList.strip()=='':
sendList="[email protected];[email protected]"
else:
sendList=sendList
toemail=sendList
to_addrs=toemail.split(';')
sendMail.sendSmail(subject,cc,fromemail,emailpasswd,to_addrs)
##执行模板:rbd-exe.py
#!/usr/bin/evn python
#-*- coding: utf-8 -*-
import sys,os,commands,subprocess
import paramiko,socket,time
import alarm_sendmail as al
import base64
reload(sys)
sys.setdefaultencoding('utf-8')
import redis,MySQLdb
def ConnRedis(rdb):
backup=rdb
host=rdb['ip']
port=rdb['port']
pool=redis.ConnectionPool(host=host,port=port,db=0,decode_responses=True)
r=redis.Redis(connection_pool=pool)
cur=conn.cursor()
if backup['CurrentDate'] !=backup['rdb_last_save_time_over'] and backup['bgsave']==0:
print "为执行bgsave.......",backup['CurrentDate'],backup['rdb_last_save_time_over']
try:
beginStart=int(time.time())
up="update db_manager.tbl_bigdate_record set bgsave=1 where ip="+"'"+str(host)+"'"+"and port="+str(port)
print "执行RDB备份中.........",beginStart
cur.execute(up)
conn.commit()
r.bgsave()
time.sleep(30)
info=r.info()
sdirName=r.config_get()['dir']
dumpName=r.config_get()['dbfilename']
rdb_back_status=''
result=''
if info['rdb_last_bgsave_status']=="ok":
rdb_back_status='ok'
result="执行RDB备份成功,完成时间:%s....."%info['rdb_last_save_time']
else:
rdb_back_status='err'
result="执行备份失败.....Codis:%s:%s"%(host,port)
print result
endup="update db_manager.tbl_bigdate_record set bgsave=0,rdb_last_save_time_over="+str(info['rdb_last_save_time'])+",rdb_last_bgsave_status="+"'"+str(rdb_back_status)+"'"+",rdb_last_save_time_begin="+str(beginStart)+" where ip="+"'"+str(host)+"'"+"and port="+str(port)
cur.execute(endup)
conn.commit()
return rdb_back_status,sdirName,dumpName
except Exception,e:
print e
elif backup['CurrentDate']==backup['rdb_last_save_time_over'] and backup['bgsave']==0 and backup['rdb_last_bgsave_status']=='err':
print "RDB Backup 异常..........Codis:%s:%s"%(host,port)
else:
rdb_back_status=3
print "RDB Backup 可以使用上次备份分析......Codis:%s:%s"%(host,port)
return rdb_back_status,1
def ConnDB(host,port,username,password,idc):
#backup={}
conn=MySQLdb.connect(host=host,port=port,user=username,passwd=password,charset='utf8')
cur=conn.cursor(MySQLdb.cursors.DictCursor)
#sql="select productName,ip,port,rdbconf,bgsave,date(from_unixtime(rdb_last_save_time_begin))as rdb_last_save_time_begin,date(from_unixtime(rdb_last_save_time_over))as rdb_last_save_time_over,rdb_last_bgsave_status,date(now())as CurrentDate from db_manager.tbl_bigdate_record"
sql="SELECT cmd.productName as productName,cmd.ip as ip,cmd.port as port,cmd.bgsave as bgsave,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_begin))AS rdb_last_save_time_begin,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_over))AS rdb_last_save_time_over,cmd.rdb_last_bgsave_status,DATE(NOW())AS CurrentDate,tmp.send_mail_to_list,pwd.password AS upassword FROM db_manager.tbl_bigdate_record cmd LEFT JOIN lepus.db_servers_redis tmp ON cmd.ip=tmp.host AND cmd.port=tmp.port LEFT JOIN dbinfo.mysql_ssh_pass pwd ON pwd.ip=cmd.ip WHERE tmp.host IS NOT NULL and cmd.idc={idc}".format(idc=str(idc))
cur.execute(sql)
backup=cur.fetchall()
# for i in data:
# for k,v in i.items():
# backup[k]=v
return backup,conn
def LoadSoft(logdir,rdb):
##下载包保存文件存储空间检查...................解压存储路径检擦
Rdbcodis=ConnRedis(rdb)
if not Rdbcodis:
print "未获取到最新得rdb文件........"
elif Rdbcodis[0]=="err":
print "RDB异常......"
elif Rdbcodis[0]==3:
print "已分析过RDB......."
else:
password_tmp=rdb['upassword']
upassword=base64.decodestring(password_tmp)
remote_file=Rdbcodis[1]
ip=rdb['ip']
productName=rdb['productName']
local_file_tmp=logdir+"/"+str(productName)
print remote_file,local_file_tmp
if not os.path.exists(local_file_tmp):
os.system(r'mkdir -p {}'.format(local_file_tmp))
remote_file=remote_file+"/"+str(Rdbcodis[2])
local_file=local_file_tmp+"/"+str(Rdbcodis[2])
ssh=paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip,22,'root',upassword)
a=ssh.exec_command('data')
stdin,stdout,stdeer=a
print stdout.read()
sftp=paramiko.SFTPClient.from_transport(ssh.get_transport())
time1=time.time()
sftp=ssh.open_sftp()
sftp.get(remote_file,local_file)
gdown=True
print "下载完成%s,Codis:%s:%s"%(str(time.time()-time1),ip,rdb['port'])
return local_file
def RdbAnalyse(logdir,rdb):
newdstDir=LoadSoft(logdir,rdb)
if not newdstDir:
print "无任何操作........"
else:
beginStart=int(time.time())
print "执行RDB分析.........",beginStart
dstDir=newdstDir
sendList=rdb['send_mail_to_list']
productName=rdb['productName']
host=rdb['ip']
port=rdb['port']
cmd_exe="/usr/local/bin/rdb -c memory -l 100 "+dstDir
dstDumpDir=os.path.dirname(dstDir)+"/"+productName+".log"
log_file=open(dstDumpDir,"w")
stdout_rdb=sys.stdout
sys.stdout=log_file
with os.popen(cmd_exe)as c:
rdb_log=c.read()
print rdb_log
log_file.close()
sys.stdout=stdout_rdb
cmddel="sed -i 1d"+" "+dstDumpDir
os.system(cmddel)
cmdr="sed -i '/^$/d'"+" "+dstDumpDir
os.system(cmdr)
endStart=int(time.time())
print "执行RDB分析完成,共计耗时:%s........."%(endStart-beginStart)
al.sendSlogLog(host,port,dstDumpDir,productName,sendList)
if __name__=="__main__":
idc=sys.argv[1]
logdir="/home/rdb-analyse/analyse"
sqlbackup_tmp=ConnDB('1.1.1.1',123,'tet','abc',idc)
conn=sqlbackup_tmp[1]
for rdb in sqlbackup_tmp[0]:
RdbAnalyse(logdir,rdb)
# else:
# print "无匹配项"
###最终执行:python rdb-exe.py number
