python抽取zabbix历史数据(1、调用zabbix的history.get接口2、直接取zabbix-mysql表，速度更快)
- 需求:轮训抽取指定hostid上单个监控项或者多个监控项的历史天数的所有数据或者抽取全部itemid
- 效果:落地到本地文件中结果如下
{
"itemid": "28885", "clock": "1585756825", "value": "97.6439", "ns": "271291797"}
{
"itemid": "28885", "clock": "1585756885", "value": "97.6375", "ns": "354267511"}
{
"itemid": "28885", "clock": "1585756945", "value": "97.6567", "ns": "432642998"}
{
"itemid": "28885", "clock": "1585757005", "value": "97.6791", "ns": "502562280"}
注意：因为写入文件中是dict转json，为了最大化的序列化速度，舍弃用标准库中的json库，使用第三方c实现的ujson，自测序列化速度提升3倍以上。
- 配置:config.ini
[zabbix_server]
url = http://192.168.1.100/zabbix/api_jsonrpc.php
user = Admin
password = zabbix
[history_get]
##文件前缀
filepre = ./item_28885
##
; 0 - numeric float;
; 1 - character;
; 2 - log;
; 3 - numeric unsigned;
; 4 - text.
history=0
##过去多少天开始
lastdays = 60
##过去多少天结束
enddays = 0
##hostid主机id组
hostids = 10264
##itemid监控项id组
itemids = 28885
#每次轮训查询的个数
limit = 10000
#是否拉取所有指标
pullall = True
- 环境:python3
- 主要依赖:requests
- 代码:main-httpapi.py
# encoding:utf-8-*
from log import Cust_Log
import configparser
import json
import requests
import os
import time
import datetime
import sys
import pymysql
import ujson
os.chdir(os.path.dirname(os.path.abspath(__file__)))
logger = Cust_Log(__file__, level="info")
class ZabbixAPI:
def __init__(self):
cf = configparser.ConfigParser()
if os.path.exists('config.ini'):
cf.read(r'config.ini')
self.__url = cf.get("zabbix_server", "url")
self.__user = cf.get("zabbix_server", "user")
self.__password = cf.get("zabbix_server", "password")
self.__header = {
"Content-Type": "application/json-rpc"}
else:
print('未读到文件config.ini')
exit()
self.__token_id = self.UserLogin()
# 登陆获取token
def UserLogin(self):
data = {
"jsonrpc": "2.0",
"method": "user.login",
"params": {
"user": self.__user,
"password": self.__password
},
"id": 0,
}
return self.PostRequest(data)
# 推送请求
def PostRequest(self, reqdata):
# print(json.dumps(reqdata))
rsp = requests.post(self.__url, headers=self.__header,
data=json.dumps(reqdata))
response = json.loads(rsp.text)
# print(response)
try:
return response['result']
except KeyError:
logger.error(response)
raise KeyError
# 主机列表
def HostGet(self, hostid=None, hostip=None):
data = {
"jsonrpc": "2.0",
"method": "host.get",
"params": {
"output": "extend",
"selectGroups": "extend",
"selectParentTemplates": ["templateid", "name"],
"selectInterfaces": ["interfaceid", "ip"],
"selectInventory": ["os"],
"selectItems": ["itemid", "name"],
"selectGraphs": ["graphid", "name"],
"selectApplications": ["applicationid", "name"],
"selectTriggers": ["triggerid", "name"],
"selectScreens": ["screenid", "name"]
},
"auth": self.__token_id,
"id": 1,
}
if hostid:
data["params"] = {
"output": "extend",
"hostids": hostid,
"sortfield": "name"
}
return self.PostRequest(data)
# 历史数据
def History(self, history, hostids, itemids, time_from, time_till, limit):
data = {
"jsonrpc": "2.0",
"method": "history.get",
"params": {
"output": "extend",
"history": history,
"hostids": hostids.split(","),
"itemids": itemids.split(","),
"time_from": time_from,
"time_till": time_till,
"sortfield": "clock",
"sortorder": "ASC",
"limit": limit
},
"auth": self.__token_id,
"id": 2
}
logger.debug("data:{}".format(data))
return self.PostRequest(data)
class MySQL:
def __init__(self, ip, port, user, password, db):
self.ip = ip
self.port = port
self.user = user
self.password = password
self.db = db
self.status = "Disconnected"
def connect(self):
self.conn = pymysql.Connect(host=self.ip, port=self.port, user=self.user,
password=self.password, database=self.db)
self.cursor = self.conn.cursor()
self.status = "Connected"
def query(self, sql):
if self.status == "Connected":
self.cursor.execute(sql)
return self.cursor.fetchall()
else:
return None
def close(self):
if self.status == "Connected":
self.cursor.close()
self.conn.close()
if __name__ == '__main__':
if os.path.exists('config.ini'):
cf = configparser.ConfigParser()
cf.read(r'config.ini')
filepre = cf.get("history_get", "filepre")
history = int(cf.get("history_get", "history"))
lastdays = int(cf.get("history_get", "lastdays"))
enddays = int(cf.get("history_get", "enddays"))
hostids = cf.get("history_get", "hostids")
itemids = cf.get("history_get", "itemids")
limit = int(cf.get("history_get", "limit"))
pullall = cf.get("history_get", "pullall")
databaseip = cf.get("zabbix_mysql", "ip")
databaseport = int(cf.get("zabbix_mysql", "port"))
databaseuser = cf.get("zabbix_mysql", "user")
databasepassword = cf.get("zabbix_mysql", "password")
databasedb = cf.get("zabbix_mysql", "defaultdb")
else:
print('未读到文件config.ini')
exit()
# 初始化zabbix API接口
api = ZabbixAPI()
start = datetime.date.today()-datetime.timedelta(days=lastdays)
startime = int(time.mktime(start.timetuple()))
end = datetime.date.today()-datetime.timedelta(days=enddays)
endtime = int(time.mktime(end.timetuple()))
# 初始化zabbix数据库
mysql = MySQL(ip=databaseip, port=databaseport,
user=databaseuser, password=databasepassword, db=databasedb)
mysql.connect()
# 查询zabbix db中itemid、hostid关系以及丰富字段，一般hostid和itemid是一对多的关系
sql = "select i.itemid, i.hostid, i.name as itemname, i.key_ as itemkey, i.status as itemstatus, h.name as hostname, h.host from items i join hosts h on i.hostid = h.hostid where i.status = 0 and i.key_ not like '%#%' and i.key_ not like '%discovery%' and h.hostid in (select hostid from interface);"
data = mysql.query(sql=sql)
if len(data) > 0:
itemDict = {
}
itemToHostID = {
}
for row in data:
itemDict[str(row[0])] = {
"hostid": row[1], "itemname": row[2], "itemkey": row[3],
"itemstatus": row[4], "hostname": row[5], "host": row[6]}
itemToHostID[str(row[0])] = row[1]
logger.debug("itemid:{},RichInfo:{}".format(
row[0], itemDict[str(row[0])]))
if len(itemDict) > 0 and pullall == "True":
itemids = ','.join([i for i in itemDict])
logger.debug("从mysql拉取的监控项itemids明细:{}".format(itemids))
logger.info("从mysql拉取的监控项itemids长度:{}".format(len(itemids)))
# 查询zabbix db中主机ip和hostid的关系，一般hostid和ip也是一对多的关系
sql = "select hostid,ip from interface;"
data = mysql.query(sql=sql)
if len(data) > 0:
hostDict = {
}
parseHostDict = {
}
for row in data:
if row[0] not in hostDict:
hostDict[row[0]] = [row[1]]
else:
_tmp = hostDict[row[0]]
logger.debug("发现hostid一对多hostip,{}:{}".format(row[0], _tmp))
_tmp.append(row[1])
hostDict[row[0]] = _tmp
logger.debug("hostid:{},ip:{}".format(
str(row[0]), hostDict[row[0]]))
logger.debug("主机id和主机IP映射关系:{}".format(hostDict))
for hostid, ips in hostDict.items():
if len(ips) == 1:
parseHostDict[hostid] = {
"hostip": ips[0]}
elif len(ips) == 0:
parseHostDict[hostid] = {
"hostip": ""}
else:
hostips = {
"hostip"+str(i+1): ip for i, ip in enumerate(ips)}
parseHostDict[hostid] = hostips
logger.debug("主机id和主机IP映射关系转换:{}".format(parseHostDict))
if len(parseHostDict) > 0 and pullall == "True":
hostids = ','.join([str(i) for i in parseHostDict])
logger.info("拉取的主机组hostids:{}".format(hostids))
# 拉取历史数据
while startime < endtime:
logger.debug(
"查询数据中time_from={}, time_till={}".format(startime, endtime))
_t1 = time.time()
rsp = api.History(history=history, hostids=hostids, itemids=itemids,
time_from=startime, time_till=endtime, limit=limit)
_t2 = time.time()
logger.info("api.History cost {} seconds".format(_t2-_t1))
# logger.debug("zabbix api history接口返回内容:{}".format(rsp))
try:
lastclock = rsp[-1]["clock"]
except:
logger.warning("最新数据无结果:{}".format(rsp))
break
if int(lastclock) == startime:
logger.info("最新时间未更新，程序退出")
break
with open(filepre+"_"+str(datetime.datetime.utcfromtimestamp(startime)), "w", 10240000)as f:
# with open(filepre+"_" + str(startime), "w")as f:
for i in rsp:
_rsp_t1 = time.time()*1000
# if i["itemid"] in itemDict:
try:
#newi = dict(i, **itemDict[i["itemid"]])
hostid = itemToHostID[i["itemid"]]
#result = dict(i, **parseHostDict[hostid])
result = dict(i, **itemDict[i["itemid"]])
# else:
except Exception as e:
logger.error(e)
logger.error(
"未丰富上主机信息,该监控项未在zabbix数据库中匹配到,itemid={}".format(i["itemid"]))
result = i
_rsp_t2 = time.time()*1000
#data = json.dumps(result, ensure_ascii=False)
data = ujson.dumps(result, ensure_ascii=False)
_rsp_t3 = time.time()*1000
f.write(data)
f.write("\n")
_rsp_t4 = time.time()*1000
logger.debug(
"rich cost %s ,transefer cost %s ,write cost %s." % (_rsp_t2-_rsp_t1, _rsp_t3-_rsp_t2, _rsp_t4-_rsp_t3))
startime = int(lastclock)+1
logger.info("查询到数据,最新clock={}\n".format(lastclock))
另一种实现通过zabbix的mysql数据库直接查询history表，相对于通过zabbix的http接口，如果你有直接访问zabbix数据库的权限，会相对更快一点。
zabbix-mysql.py
# encoding:utf-8-*
from log import Cust_Log
import configparser
import json
import requests
import os
import time
import datetime
import sys
import pymysql
import ujson
os.chdir(os.path.dirname(os.path.abspath(__file__)))
logger = Cust_Log(__file__, level="info")
class MySQL:
def __init__(self, ip, port, user, password, db):
self.ip = ip
self.port = port
self.user = user
self.password = password
self.db = db
self.status = "Disconnected"
def connect(self):
self.conn = pymysql.Connect(host=self.ip, port=self.port, user=self.user,
password=self.password, database=self.db)
self.cursor = self.conn.cursor()
self.status = "Connected"
def query(self, sql):
if self.status == "Connected":
self.cursor.execute(sql)
return self.cursor.fetchall()
else:
return None
def close(self):
if self.status == "Connected":
self.cursor.close()
self.conn.close()
if __name__ == '__main__':
if os.path.exists('config.ini'):
cf = configparser.ConfigParser()
cf.read(r'config.ini')
filepre = cf.get("history_get", "filepre")
history = int(cf.get("history_get", "history"))
lastdays = int(cf.get("history_get", "lastdays"))
enddays = int(cf.get("history_get", "enddays"))
hostids = cf.get("history_get", "hostids")
itemids = cf.get("history_get", "itemids")
limit = int(cf.get("history_get", "limit"))
pullall = cf.get("history_get", "pullall")
databaseip = cf.get("zabbix_mysql", "ip")
databaseport = int(cf.get("zabbix_mysql", "port"))
databaseuser = cf.get("zabbix_mysql", "user")
databasepassword = cf.get("zabbix_mysql", "password")
databasedb = cf.get("zabbix_mysql", "defaultdb")
else:
print('未读到文件config.ini')
exit()
start = datetime.date.today()-datetime.timedelta(days=lastdays)
startime = int(time.mktime(start.timetuple()))
end = datetime.date.today()-datetime.timedelta(days=enddays)
endtime = int(time.mktime(end.timetuple()))
# 初始化zabbix数据库
mysql = MySQL(ip=databaseip, port=databaseport,
user=databaseuser, password=databasepassword, db=databasedb)
mysql.connect()
# 查询zabbix db中itemid、hostid关系以及丰富字段，一般hostid和itemid是一对多的关系
sql = "select i.itemid, i.hostid, i.name as itemname, i.key_ as itemkey, i.status as itemstatus, h.name as hostname, h.host from items i join hosts h on i.hostid = h.hostid where i.status = 0 and i.key_ not like '%#%' and i.key_ not like '%discovery%' and h.hostid in (select hostid from interface);"
data = mysql.query(sql=sql)
if len(data) > 0:
itemDict = {
}
itemToHostID = {
}
for row in data:
itemDict[str(row[0])] = {
"hostid": row[1], "itemname": row[2], "itemkey": row[3],
"itemstatus": row[4], "hostname": row[5], "host": row[6]}
itemToHostID[str(row[0])] = row[1]
logger.debug("itemid:{},RichInfo:{}".format(
row[0], itemDict[str(row[0])]))
if len(itemDict) > 0 and pullall == "True":
itemids = ','.join([i for i in itemDict])
logger.debug("从mysql拉取的监控项itemids明细:{}".format(itemids))
logger.info("从mysql拉取的监控项itemids长度:{}".format(len(itemids)))
# 查询zabbix db中主机ip和hostid的关系，一般hostid和ip也是一对多的关系
sql = "select hostid,ip from interface;"
data = mysql.query(sql=sql)
if len(data) > 0:
hostDict = {
}
parseHostDict = {
}
for row in data:
if row[0] not in hostDict:
hostDict[row[0]] = [row[1]]
else:
_tmp = hostDict[row[0]]
logger.debug("发现hostid一对多hostip,{}:{}".format(row[0], _tmp))
_tmp.append(row[1])
hostDict[row[0]] = _tmp
logger.debug("hostid:{},ip:{}".format(
str(row[0]), hostDict[row[0]]))
logger.debug("主机id和主机IP映射关系:{}".format(hostDict))
for hostid, ips in hostDict.items():
if len(ips) == 1:
parseHostDict[hostid] = {
"hostip": ips[0]}
elif len(ips) == 0:
parseHostDict[hostid] = {
"hostip": ""}
else:
hostips = {
"hostip"+str(i+1): ip for i, ip in enumerate(ips)}
parseHostDict[hostid] = hostips
logger.debug("主机id和主机IP映射关系转换:{}".format(parseHostDict))
if len(parseHostDict) > 0 and pullall == "True":
hostids = ','.join([str(i) for i in parseHostDict])
logger.info("拉取的主机组hostids:{}".format(hostids))
# 拉取历史数据
while startime < endtime:
with open(filepre+"_"+str(datetime.datetime.utcfromtimestamp(startime)), "w", 10240000)as f:
logger.debug(
"查询数据中time_from={}, time_till={}".format(startime, endtime))
_t1 = time.time()
sql = "select itemid,clock,value,ns from history where clock>{} limit {};".format(
startime, limit)
data = mysql.query(sql=sql)
_t2 = time.time()
logger.info(
"Query MySQL History Table Cost {} seconds".format(_t2-_t1))
if len(data) > 0:
logger.debug("get response data")
for row in data:
new = {
"itemid": row[0], "clock": row[1],
"value": row[2], "ns": row[3]}
new1 = dict(new, **itemDict[str(row[0])])
hostid = itemToHostID[str(row[0])]
new2 = dict(new1, **parseHostDict[hostid])
newrow = ujson.dumps(new2, ensure_ascii=False)
f.write(newrow)
f.write("\n")
try:
lastclock = data[-1][1]
except:
logger.warning("最新数据无结果:{}".format(rsp))
break
if int(lastclock) == startime:
logger.info("最新时间未更新，程序退出")
break
# sys.exit(1)
startime = int(lastclock)+1
logger.info("查询到数据,最新clock={}\n".format(lastclock))