Python 自动化管理Mysql数据库
MySQL数据库一款非常优秀的开源数据库，很多人都在使用。我也不例外，数据库实例创建、数据库状态检测、数据库备份等，天天做着重复、枯燥的工作。为了减轻工作量，使用Python写了一个自动管理Mysql数据库的工具。
具体功能：
Mysql管理工具的主要功能：
> 数据库实例创建
> 数据库备份
> 数据库配置检测
> 数据库主从同步
> 配置文件重新载入
2. 工具代码结构：
library: 将共用的功能封装成一个库。
mysqlmanager：myman.py脚本实现基本管理Mysql的功能。
3. 工具代码展示
library/mysql.py:
#!/usr/local/bin/python2.7
#-*- coding:utf-8 -*-
from ConfigParser import ConfigParser
import os
import MySQLdb
def getMyVariables(cur):
'''查询数据库配置信息'''
cur.execute('show global variables;')
data = cur.fetchall()
return dict(data)
class MySQLDConfig(ConfigParser):
'''将所有公用的功能封装成一个class'''
def __init__(self, config, **kw):
'''Python版本必须2.7以上，2.6版本没有allow_no_value 属性'''
ConfigParser.__init__(self, allow_no_value=True)
self.config = config
self.mysqld_vars = {}
if os.path.exists(self.config):
self.read(self.config)
self.get_mysqld_vars()
else:
self.set_mysqld_defaults_var()
self.set_mysqld_vars(kw)
def set_mysqld_vars(self, kw):
'''获取配置文件信息，覆盖默认配置'''
for k, v in kw.items():
self.mysqld_vars[k] = v
def get_mysqld_vars(self):
'''获取现有配置文件信息'''
options = self.options('mysqld')
rst = {}
for o in options:
rst[o] = self.get('mysqld', o)
self.set_mysqld_vars(rst)
def set_mysqld_defaults_var(self):
'''如果配置文件不存在，设置默认配置'''
defaults = {
"user":"mysql",
"pid-file": "/var/run/mysqld/mysqld.pid",
"socket": "/var/lib/mysql/mysql.sock",
"port": "3306",
"basedir": "/usr",
"datadir": "/tmp/mysql",
"tmpdir": "/tmp",
"skip-external-locking": None,
"bind-address": "127.0.0.1",
"key_buffer": "16M",
"max_allowed_packet": "16M",
"thread_stack": "192K",
"thread_cache_size": "8",
"myisam-recover": "BACKUP",
"query_cache_limit": "1M",
"query_cache_size": "16M",
"log_error": "/var/log/mysqld.log",
"expire_logs_days": "10",
"max_binlog_size": "100M"
}
self.set_mysqld_vars(defaults)
def save(self):
'''将配置信息保存至配置文件'''
if not self.has_section('mysqld'):
self.add_section('mysqld')
for k, v in self.mysqld_vars.items():
self.set('mysqld', k, v)
with open(self.config, 'w') as fd:
self.write(fd)
if __name__ == "__main__":
mc = MySQLDConfig('/root/david/mysqlmanager/cnfs/my.cnf', max_connection=200, user='mysql')
mc.set_var('skip-slave-start', None)
mc.save()
library/utils.py
#!/usr/local/bin/python2.7
#-*-coding:utf-8 -*-
'''
格式时间的转换，数据库配置文件的单位(*.cnf)和数据库global（mysql>show global variables;）配置的单位不一致，需要转换
'''
unit = {'t':2**40,'g':2**30,'m':2**20,'k':2**10,'b':1}
def convertUnit(s):
s = s.lower()
lastchar = s[-1]
num = int(s[:-1])
if lastchar in unit:
return num*unit[lastchar]
else:
return int(s)
def scaleUnit(d):
for k,v in unit.items():
num = d / v
if (0 < num < 2**10):
return num,k
mysqlmanager/myman.py:
#!/usr/local/bin/python2.7
#-*- coding:utf-8 -*-
from os import path
from optparse import OptionParser
from subprocess import PIPE, Popen
import MySQLdb
import glob
import os
import sys
import time
import datetime
import re
DIRNAME = path.dirname(__file__)
OPSTOOLS_DIR = path.abspath(path.join(DIRNAME, '..'))
sys.path.append(OPSTOOLS_DIR)
from library.mysql import MySQLDConfig, getMyVariables
REPLICATION_USER = 'repl'
REPLICATION_PASS = '123qwe'
MYSQL_DATA_DIR = '/home/david/data'
MYSQL_CONF_DIR = '/home/david/cnfs'
MYSQL_BACK_DIR = '/home/david/backup'
def opts():
parser = OptionParser(usage="usage: %prog [options] arg1 arg2")
parser.add_option("-c","--cmd",
dest="cmd",
action="store",
default="check",
help="Check the configuration file and database configuration parameters are different.[%options]"
)
parser.add_option("-n","--name",
dest="name",
action="store",
default="mysqlinstance",
help="Create Examples."
)
parser.add_option("-p","--port",
dest="port",
action="store",
default="3306",
help="Examples of port."
)
return parser.parse_args()
def checkPort(d, p):
'''实例端口检测'''
for m in d:
if p == m.mysqld_vars['port']:
return True
return False
def setReplMaster(cur):
'''设置slave数据库同步用户的授权'''
sql = "GRANT REPLICATION SLAVE ON *.* TO %s@'localhost' IDENTIFIED BY '%s'" % (REPLICATION_USER, REPLICATION_PASS)
cur.execute(sql)
def connMySQLd(mc):
'''连接数据库'''
host = '127.0.0.1'
user = 'root'
port = int(mc.mysqld_vars['port'])
conn = MySQLdb.connect(host, port=port, user=user)
cur = conn.cursor()
return cur
def run_mysql(cnf):
'''运行数据库'''
cmd = "mysqld_safe --defaults-file=%s &" % cnf
p = Popen(cmd, stdout=PIPE, shell=True)
time.sleep(5)
return p.returncode
def setOwner(p, user):
'''设置目录权限'''
os.system("chown -R %s:%s %s" % (user, user, p))
def mysql_install_db(cnf):
'''数据库初始化'''
p = Popen("mysql_install_db --defaults-file=%s" % cnf, stdout=PIPE, shell=True)
#p = Popen("mysql_install_db --user=mysql --datadir=%s " % MYSQL_DATA_DIR, stdout=PIPE, shell=True)
stdout, stderr = p.communicate()
return p.returncode
def _genDict(name, port):
'''设置文件存储目录及监听端口'''
return {
'pid-file': path.join(MYSQL_DATA_DIR, name, "%s.pid" % name),
'socket': '/tmp/%s.sock' % name,
'port': port,
'datadir': path.join(MYSQL_DATA_DIR, name)+'/',
'log_error': path.join(MYSQL_DATA_DIR, name)
}
def readConfs():
'''读取配置文件，如果配置文件不存在，使用默认配置生成配置文件'''
confs = glob.glob(path.join(MYSQL_CONF_DIR, '*.cnf'))
return [MySQLDConfig(c) for c in confs]
def getCNF(name):
'''获取配置文件完整路径'''
return path.join(MYSQL_CONF_DIR, "%s.cnf" % name)
def runMySQLdump(cmd):
'''启动Mysql命令'''
p = Popen(cmd, stdout=PIPE, shell=True)
stdout, stderr = p.communicate()
return p.returncode
def getBinlogPOS(f):
'''获取binlog'''
with open(f) as fd:
f, p = findLogPos(l)
if f and p:
return f,p
def findLogPos(s):
rlog = re.compile(r"MASTER_LOG_FILE='(\S+)',", re.IGNORECASE)
rpos = re.compile(r"MASTER_LOG_POS=(\d+),?", re.IGNORECASE)
log = rlog.search(s)
pos = rpos.search(s)
if log and pos:
return log.group(1), int(pos.group(1))
else:
return (None, None)
def changeMaster(cur, host, port, user, mpass, mf, p):
sql = '''CHANGE MASTER TO
MASTER_HOST='%s',
MASTER_PORT='%s',
MASTER_USER='%s',
MASTER_PASSWORD='%s',
MASTER_LOG_FILE='%s',
MASTER_LOG_POS=%s;''' % (host, port, user, mpass, mf, p)
cur.execute(sql)
def createInstance(name, port, dbtype="master", **kw):
'''创建数据库实例'''
cnf = path.join(MYSQL_CONF_DIR, "%s.cnf" % name)
datadir = path.join(MYSQL_DATA_DIR, name)
exists_cnfs = readConfs()
if checkPort(exists_cnfs, port):
print >> sys.stderr, "port exist."
sys.exit(-1)
if not path.exists(cnf):
c = _genDict(name, port)
c.update(kw)
mc = MySQLDConfig(cnf, **c)
mc.save()
else:
mc = MySQLDConfig(cnf, **kw)
if not path.exists(datadir):
mysql_install_db(cnf)
setOwner(datadir, mc.mysqld_vars['user'])
run_mysql(cnf)
time.sleep(3)
cur = connMySQLd(mc)
setReplMaster(cur)
def diffVariables(instance_name):
'''查询数据库配置文件和数据库配置的差异'''
cnf = getCNF(instance_name)
if path.exists(cnf):
mc = MySQLDConfig(cnf)
print mc
cur = connMySQLd(mc)
vars = getMyVariables(cur)
for k, v in mc.mysqld_vars.items():
k = k.replace('-', '_')
if k in vars and vars[k] != v:
print k, v, vars[k]
def setVariable(instance_name, variable, value):
'''重新加载配置'''
cnf = getCNF(instance_name)
if path.exists(cnf):
mc = MySQLDConfig(cnf)
cur = connMySQLd(mc)
cur.execute('set global %s = %s' % (variable, value))
mc.set_var(variable, value)
mc.save()
def backupMySQL(instance_name):
'''备份数据库'''
cnf = getCNF(instance_name)
if path.exists(cnf):
mc = MySQLDConfig(cnf)
now = datetime.datetime.now()
timestamp = now.strftime('%Y-%m-%d-%H%M%S')
backup_file = path.join(MYSQL_BACK_DIR, instance_name, timestamp+'.sql')
_dir = path.dirname(backup_file)
if not path.exists(_dir):
os.makedirs(_dir)
cmd = 'mysqldump -A -x -F --master-data=1 --host=127.0.0.1 --user=root --port=%s > %s' % (mc.mysqld_vars['port'], backup_file)
runMySQLdump(cmd)
def restoreMySQL(instance_name, instance_port, sqlfile, **kw):
createInstance(instance_name, instance_port, **kw)
cnf = getCNF(instance_name)
if path.exists(cnf):
mc = MySQLDConfig(cnf)
cur = connMySQLd(mc)
cmd = "mysql -h 127.0.0.1 -P %s -u root < %s" % (mc.mysqld_vars['port'], sqlfile)
f, p = getBinlogPOS(sqlfile)
runMySQLdump(cmd)
changeMaster(cur,
host=kw['master-host'],
port=kw['master-port'],
user=REPLICATION_USER,
mpass=REPLICATION_PASS,
mf=f,
p=p)
def _init():
'''查询mysql几个目录是否存在，如果不存在，自动创建'''
if not path.exists(MYSQL_DATA_DIR):
os.makedirs(MYSQL_DATA_DIR)
if not path.exists(MYSQL_CONF_DIR):
os.makedirs(MYSQL_CONF_DIR)
if not path.exists(MYSQL_BACK_DIR):
os.makedirs(MYSQL_BACK_DIR)
def main():
opt, args = opts()
instance_name = opt.name
instance_port = opt.port
command = opt.cmd
if command == "create":
if not args:
createInstance(instance_name, instance_port)
else:
dbtype = args[0]
serverid = args[1]
mysqld_options = {'server-id':serverid}
if dbtype == 'master':
mysqld_options['log-bin'] = 'mysql-bin'
elif dbtype == 'slave':
master_host = args[2]
master_port = args[3]
mysqld_options['master-host'] = master_host
mysqld_options['master-port'] = master_port
mysqld_options['master-user'] = REPLICATION_USER
mysqld_options['master-password'] = REPLICATION_PASS
mysqld_options['skip-slave-start'] = None
mysqld_options['replicate-ignore-db'] = 'mysql'
mysqld_options['read-only'] = None
createInstance(instance_name, instance_port, dbtype=dbtype, **mysqld_options)
elif command == 'check':
diffVariables(instance_name)
elif command == 'adjust':
variable = args[0]
value = args[1]
setVariable(instance_name, variable, value)
elif command == 'backup':
backupMySQL(instance_name)
elif command == 'restore':
serverid == args[0]
mhost = args[1]
mport = args[2]
sqlfile = args[3]
mysqld_options = {
"master-host":mhost,
"master-port":mport,
"server-id":serverid,
"skip-slave-start":None,
}
restoreMySQL(instance_name, instance_port, sqlfile, **mysqld_options)
if __name__ == "__main__":
print main()
4.测试
帮助信息：
创建master实例：
创建slave实例：
检测配置文件和数据库加载配置差异：
由于单位格式不同，所以出现了差异,可以结合library/utils.py 进行单位换算
数据库备份：
需要注意：
1. python版本必须2.7及2.7以上版本。
2. MYSQL_DATA_DIR 目录不能放在/root目录下，如果放在root目录下，初始化数据库的时候会报错(权限问题)。我在这犯过错.
如果哪里有错误，或者不足的地方。还请大家多多沟通。