自动运维.编写线上P2P设备状态查询服务器StatusServer部分功能预警测试?
具体要求:
从国内RDS数据库中读取指定账户下的设备列表
通过Docker中的手机端StatusServer接口批量设备状态(每次申请授权码时先查询是否已经授权,如果没授权调用授权接口重新授权)
定义错误信息写入本区域内的Redis数据库,后端会LPOP判断是否调用微信报警接口向订阅者推送预警信息
文件结构:
代码实现:
/xm-workspace/xm-pyss/auto_python/check_StatusServer/readme.txt
Running: /usr/bin/python main.py
/xm-workspace/xm-pyss/auto_python/check_StatusServer/comdata.py:
# -*- coding: utf-8 -*-
"""
#
# Authors: limanman
# OsChina: http://my.oschina.net/pydevops/
# Purpose:
#
"""
# 授权HTTP请求头
auth_headers = {
"CSeq": "1",
"Host": "...",
"Port": ...,
"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)"
}
# 授权HTTP请求体
auth_qryreq = {
u'AuthProtocol': {
u'Body': {
u'Attitude': u'...',
u'SecretKey': u'...',
u'SerialNumber': u'',
u'UserName': u'test'},
u'Header': {
u'CSeq': u'1',
u'MessageType': u'MSG_AUTH_CODE_QUERY_REQ',
u'Version': u'1.0'}}}
auth_newreq = {
u'AuthProtocol': {
u'Body': {
u'Attitude': u'...',
u'SecretKey': u'...',
u'SerialNumber': u'...',
u'UserName': u'test'},
u'Header': {u'CSeq': u'1',
u'MessageType': u'MSG_AUTH_CODE_NEW_REQ',
u'Version': u'1.0'}}}
# 查询HTTP请求头
zero_headers = {
"CSeq": "1",
"Host": "...",
"Port": ...,
"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)"
}
# 查询HTTP请求体
zero_query = {
u'StatusProtocol': {
u'Body': [],
u'Header': {
u'CSeq': u'1',
u'MessageType': u'MSG_STATUS_MULTIQUERY_REQ',
u'Version': u'1.0'}}}
# 连接池相关设置
mysql_conf = {
'port': 3306,
# 启动时连接池中创建的的连接数
'mincached': 5,
# 连接池中最大允许创建的连接数
'maxcached': 20,
'user': '.....',
'charset': 'utf8',
'db': '.........',
# 所有允许的最大连接数上限设置
'maxconnections': 62,
'passwd': '................',
'host': '...................'
}
# 查询用户下设备
query_device = r"select mac from device_my where userid=(select id from `user` where username='...');"
if __name__ == '__main__':
print 'Found Notice: the module can only be import.'
/xm-workspace/xm-pyss/auto_python/check_StatusServer/daemon.py:
# -*- coding: utf-8 -*-
"""
#
# Authors: limanman
# OsChina: http://my.oschina.net/pydevops/
# Purpose:
#
"""
import os
import sys
import time
# 守护进程
def daemonize(stdin=os.devnull, stdout=os.devnull, stderr=os.devnull):
"""Fork current process as daemon process.
Args:
stdin : stand input
stdout: stand output
stderr: stand error
Returns:
None
"""
try:
cur_pid = os.fork()
if cur_pid > 0:
sys.exit(0)
except OSError as fork_err:
sys.stderr.write('Found Errors: fork %s failed, %s' % (cur_pid, fork_err.strerror))
sys.exit(1)
os.chdir('/')
os.umask(0)
os.setsid()
try:
nxt_pid = os.fork()
if nxt_pid > 0:
sys.exit(0)
except OSError as fork_err:
sys.stderr.write('Found Errors: fork %s failed, %s' % (cur_pid, fork_err.strerror))
sys.exit(1)
# flush stdin, stdout, stderr
for f in (sys.stdin, sys.stdout, sys.stderr):
f.flush()
f_stdin = open(stdin, 'r')
f_stdout = open(stdout, 'a+')
f_stderr = open(stderr, 'a+', 0)
os.dup2(f_stdin.fileno(), sys.stdin.fileno())
os.dup2(f_stdout.fileno(), sys.stdout.fileno())
os.dup2(f_stderr.fileno(), sys.stderr.fileno())
/xm-workspace/xm-pyss/auto_python/check_StatusServer/pyutils.py:
# -*- coding: utf-8 -*-
"""
#
# Authors: limanman
# OsChina: http://my.oschina.net/pydevops/
# Purpose:
#
"""
import sys
import json
import redis
import httplib
import MySQLdb
# 导入数据库连接池
from comdata import mysql_conf
from DBUtils.PooledDB import PooledDB
# 数据库连接池类
class Mysql(object):
"""MySQL connection pool class.
"""
__mysql_pool = None
def __init__(self):
# 获取连接对象
self._mysql_conn = Mysql.__get_connection()
# 创建游标对象
self._mysql_cursor = self._mysql_conn.cursor()
@staticmethod
def __get_connection():
"""Get a connection from connection pool.
Args:
None
Returns:
connection object
"""
# 根据配置文件创建连接池
if not Mysql.__mysql_pool:
Mysql.__mysql_pool = PooledDB(
creator=MySQLdb,
use_unicode=False,
db=mysql_conf['db'],
host=mysql_conf['host'],
port=mysql_conf['port'],
user=mysql_conf['user'],
passwd=mysql_conf['passwd'],
charset=mysql_conf['charset'],
mincached=mysql_conf['mincached'],
maxcached=mysql_conf['maxcached'],
maxconnections=mysql_conf['maxconnections'])
# 返回连接池中连接对象
return Mysql.__mysql_pool.connection()
def get_all(self, sql_command, cmd_param=None):
""""Get all results of sql command execute.
Args:
sql_command: sql command
cmd_param : sql command paramaters
Returns:
tuple
"""
if cmd_param:
count = self._mysql_cursor.execute(sql_command, cmd_param)
else:
count = self._mysql_cursor.execute(sql_command)
if count:
sql_result = self._mysql_cursor.fetchall()
else:
sql_result = None
return sql_result
def release(self):
# 释放刚获取的连接,并没有关闭连接
self._mysql_cursor.close()
self._mysql_conn.close()
# HTTP发包类
class Http(object):
"""
Args:
http_args: http arguements
"""
def __init__(self, **headers):
self.headers = headers
def post_data(self, httpdata, uri=''):
"""Post http data.
Args:
httpdata: http data
Returns:
dict/None
"""
Host = self.headers['Host']
Port = self.headers['Port']
req_url = 'http://%s:%s/%s' % (Host, Port, uri)
connect = httplib.HTTPConnection(Host, Port)
connect.request('POST', url=req_url, body=httpdata, headers=self.headers)
response = connect.getresponse()
return response.read()
# 数据库连接池类
class Redis(object):
"""Redis connection pool class.
Attributes:
host: redis host ip
port: redis port
"""
def __init__(self, host='127.0.0.1', port=5123):
self.pool = redis.ConnectionPool(host=host, port=port)
self.conn = redis.StrictRedis(connection_pool=self.pool)
def redis_lpush(self, redis_value, redis_list='AlarmServerList'):
"""Push the value to the head of AlarmServerList.
Args:
redis_value: alarm message
redis_list : redis list key, default AlarmServerList
"""
return self.conn.lpush(redis_list, redis_value)
if __name__ == '__main__':
print 'Found Notice: the module can only be imported, not run directly.'
/xm-workspace/xm-pyss/auto_python/check_StatusServer/check_statusserver.py:
# -*- coding: utf-8 -*-
"""
#
# Authors: limanman
# OsChina: http://my.oschina.net/pydevops/
# Purpose:
#
"""
import os
import sys
import json
import time
import random
import daemon
import comdata
import pyutils
import datetime
import dns.resolver
# DNS查询(可能是多个)
def dns_query(domain_name):
"""Query domain A record.
Args:
domain_name: domain name
Returns:
str
"""
dns_list = []
dns_resolver = dns.resolver.Resolver()
dns_answer = dns_resolver.query(domain_name, 'A')
for cur_answer in dns_answer.response.answer:
for cur_item in cur_answer.items:
dns_list.append(cur_item.address)
return dns_list[0]
# 推送消息(默认不上报)
def push(cur_pid, dst_addr, err_msg, err_level='Notice', err_push=False):
"""Lpush message to redis AlarmServerList.
Args:
dst_addr : detected dest ip address
err_msg : error messages
err_level: error level, default Notice
err_push : whether to push message, default False
Returns:
str
"""
srv_name = 'StatusServer'
rand_num = random.randint(1111111111, 9999999999)
date_time = datetime.datetime.today().__str__()[:19]
res_strs = '%s %s %s %s %s %s %s\n' % (date_time, rand_num,
srv_name, err_level,
cur_pid, dst_addr, err_msg)
if err_push:
redis_conn = pyutils.Redis()
redis_conn.redis_lpush(res_strs)
sys.stderr.write(res_strs)
sys.stderr.flush()
# 发送请求(公用方法)
def post(post_data, **headers):
"""Post url data
Args:
post_data: strs url data
headers : header dict
Returns:
None
"""
httpconn = pyutils.Http(**headers)
common_data = json.dumps(post_data)
http_response = httpconn.post_data(common_data)
response_dict = json.loads(http_response)
return response_dict
# 入口函数
def main(check_interval):
"""Check device whether online?
Args:
check_interval: check device status interval
Returns:
None
"""
sqls_addr = dns_query('clouddata.mysql.rds.aliyuncs.com')
auth_addr = dns_query(comdata.auth_headers['Host'])
zero_addr = dns_query(comdata.zero_headers['Host'])
while True:
# 获取所有设备列表
try:
sql_conn = pyutils.Mysql()
except BaseException as mysql_errs:
push(PROCESS_ID, sqls_addr, mysql_errs[-1], 'High', True)
time.sleep(check_interval)
continue
sql_rets = sql_conn.get_all(comdata.query_device)
sql_rets = zip(*sql_rets)[0]
# 释放连接池的连接
sql_conn.release()
# 获取设备和授权码
device_list = []
autherrflag = False
for device_id in sql_rets:
comdata.auth_qryreq['AuthProtocol']['Body']['SerialNumber'] = device_id
try:
http_response = post(comdata.auth_qryreq, **comdata.auth_headers)
except BaseException as socket_errs:
push(PROCESS_ID, auth_addr, socket_errs, 'High', True)
time.sleep(check_interval)
autherrflag = True
break
try:
device_authcode = http_response['AuthProtocol']['Body']['AuthCode']
except KeyError:
device_authcode = None
if not device_authcode:
comdata.auth_newreq['AuthProtocol']['Body']['SerialNumber'] = device_id
http_response = post(comdata.auth_newreq, **comdata.auth_headers)
try:
device_authcode = http_response['AuthProtocol']['Body']['AuthCode']
except KeyError:
auth_errs = http_response['AuthProtocol']['Header']['ErrorString']
push(PROCESS_ID, auth_addr, auth_errs, 'High', True)
autherrflag = True
break
device_list.append({u'AuthCode': device_authcode, u'SerialNumber': device_id})
if autherrflag:
time.sleep(check_interval)
continue
# 批量查询设备状态
comdata.zero_query['StatusProtocol']['Body'][:] = device_list
try:
http_response = post(comdata.zero_query, **comdata.zero_headers)
except BaseException as socket_errs:
push(PROCESS_ID, zero_addr, socket_errs, 'High', True)
time.sleep(check_interval)
continue
device_items = http_response['StatusProtocol']['Body']
alarmerrflag = False
for cur_device in device_items:
if cur_device['Status'] == 'Online':
alarmerrflag = True
break
if not alarmerrflag:
status_err = 'all device under the user ytest is offline.'
push(PROCESS_ID, zero_addr, status_err, 'High', True)
# 每n分钟检测一次
time.sleep(check_interval)
if __name__ == '__main__':
PROCESS_ID = os.getpid()
# 脱离终端后台运行
stderr_log = os.path.join(os.getcwd(), 'statusserver_err.log')
stdout_log = os.path.join(os.getcwd(), 'statusserver_out.log')
daemon.daemonize(stdin=os.devnull, stdout=stdout_log, stderr=stderr_log)
# 检测时间为3分钟
main(180)