【CMDB】第3章——CMDB客户端开发
客户端程序目录结构：
程序入口
NedStark.py：
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
import os,sys,platform
#for linux
if platform.system() == "Windows":
BASE_DIR = '\\'.join(os.path.abspath(os.path.dirname(__file__)).split('\\')[:-1])
print(BASE_DIR)
else:
BASE_DIR = '/'.join(os.path.abspath(os.path.dirname(__file__)).split('/')[:-1])
sys.path.append(BASE_DIR)
from core import HouseStark #main
if __name__ == '__main__':
HouseStark.ArgvHandler(sys.argv)
根据传入的不同参数执行不同的方法（以收集数据为例）
HouseStark.py
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
from core import info_collection
from conf import settings
import urllib.request,sys,os,json,datetime
import urllib.parse
from core import api_token
class ArgvHandler(object):
def __init__(self,argv_list):
self.argvs = argv_list
self.parse_argv()
def parse_argv(self):
if len(self.argvs) >1:
if hasattr(self,self.argvs[1]):
func = getattr(self,self.argvs[1])
func()
else:
self.help_msg()
else:
self.help_msg()
def help_msg(self):
msg = '''
collect_data 收集资产数据
run_forever 未实现
get_asset_id 获取资产id
report_asset 汇报资产数据到服务器
'''
print(msg)
def collect_data(self):
"""
收集数据
:return:
"""
obj = info_collection.InfoCollection()
asset_data = obj.collect() #收集
print(asset_data)
def run_forever(self):
pass
def __attach_token(self,url_str):
'''generate md5 by token_id and username,and attach it on the url request'''
user = settings.Params['auth']['user']
token_id = settings.Params['auth']['token']
md5_token,timestamp = api_token.get_token(user,token_id)
url_arg_str = "user=%s×tamp=%s&token=%s" %(user,timestamp,md5_token)
if "?" in url_str:#already has arg
new_url = url_str + "&" + url_arg_str
else:
new_url = url_str + "?" + url_arg_str
return new_url
#print(url_arg_str)
def __submit_data(self,action_type,data,method):
'''
send data to server
:param action_type: url
:param data: 具体要发送的数据
:param method: get/post
:return:
'''
if action_type in settings.Params['urls']:
if type(settings.Params['port']) is int:
url = "http://%s:%s%s" %(settings.Params['server'],settings.Params['port'],settings.Params['urls'][action_type])
else:
url = "http://%s%s" %(settings.Params['server'],settings.Params['urls'][action_type])
url = self.__attach_token(url)
print('Connecting [%s], it may take a minute' % url)
if method == "get":
args = ""
for k,v in data.items():
args += "&%s=%s" %(k,v)
args = args[1:]
url_with_args = "%s?%s" %(url,args)
print(url_with_args)
try:
req = urllib.request.urlopen(url_with_args,timeout=settings.Params['request_timeout'])
#req_data =urlopen(req,timeout=settings.Params['request_timeout'])
#callback = req_data.read()
callback = req.read()
print("-->server response:",callback)
return callback
except urllib.URLError as e:
sys.exit("\033[31;1m%s\033[0m"%e)
elif method == "post":
try:
data_encode = urllib.parse.urlencode(data).encode()
req = urllib.request.urlopen(url=url,data=data_encode,timeout=settings.Params['request_timeout'])
#res_data = urllib.urlopen(req,timeout=settings.Params['request_timeout'])
callback = req.read()
callback = json.loads(callback.decode())
print("\033[31;1m[%s]:[%s]\033[0m response:\n%s" %(method,url,callback) )
return callback
except Exception as e:
sys.exit("\033[31;1m%s\033[0m"%e)
else:
raise KeyError
#def __get_asset_id_by_sn(self,sn):
# return self.__submit_data("get_asset_id_by_sn",{"sn":sn},"get")
def load_asset_id(self,sn=None):
asset_id_file = settings.Params['asset_id']
has_asset_id = False
if os.path.isfile(asset_id_file):
asset_id = open(asset_id_file).read().strip()
if asset_id.isdigit():
return asset_id
else:
has_asset_id = False
else:
has_asset_id = False
def __update_asset_id(self,new_asset_id):
asset_id_file = settings.Params['asset_id']
f = open(asset_id_file,"w",encoding="utf-8")
f.write(str(new_asset_id))
f.close()
def report_asset(self):
obj = info_collection.InfoCollection()
asset_data = obj.collect()
asset_id = self.load_asset_id(asset_data["sn"])
if asset_id: #reported to server before
asset_data["asset_id"] = asset_id
post_url = "asset_report"
else:#first time report to server
'''report to another url,this will put the asset into approval waiting zone, when the asset is approved ,this request returns
asset's ID'''
asset_data["asset_id"] = None
post_url = "asset_report_with_no_id"
data = {"asset_data": json.dumps(asset_data)}
response = self.__submit_data(post_url,data,method="post")
if "asset_id" in response:
self.__update_asset_id(response["asset_id"])
self.log_record(response)
def log_record(self,log,action_type=None):
f = open(settings.Params["log_file"],"ab")
if log is str:
pass
if type(log) is dict:
if "info" in log:
for msg in log["info"]:
log_format = "%s\tINFO\t%s\n" %(datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S"),msg)
#print msg
f.write(log_format.encode())
if "error" in log:
for msg in log["error"]:
log_format = "%s\tERROR\t%s\n" %(datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S"),msg)
f.write(log_format.encode())
if "warning" in log:
for msg in log["warning"]:
log_format = "%s\tWARNING\t%s\n" %(datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S"),msg)
f.write(log_format.encode())
f.close()
执行具体的收集方法
info_collection.py
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
from plugins import plugin_api
import json,platform,sys
class InfoCollection(object):
'''收集硬件信息'''
def __init__(self):
pass
def get_platform(self):
“”“
获取平台
”“”
os_platform = platform.system()
return os_platform
def collect(self):
os_platform = self.get_platform()
try:
func = getattr(self,os_platform)
info_data = func()
formatted_data = self.build_report_data(info_data)
return formatted_data
except AttributeError as e:
sys.exit("Error:MadKing doens't support os [%s]! " % os_platform)
def Linux(self):
sys_info = plugin_api.LinuxSysInfo()
return sys_info
def Windows(self):
sys_info = plugin_api.WindowsSysInfo()
#print(sys_info)
#f = file('data_tmp.txt','wb')
#f.write(json.dumps(sys_info))
#f.close()
return sys_info
def build_report_data(self,data):
#add token info in here before send
return data
不同平台选择不同的收集数据方式
plugin_api.py
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
from plugins.linux import sysinfo
def LinuxSysInfo():
#print __file__
return sysinfo.collect()
def WindowsSysInfo():
from plugins.windows import sysinfo as win_sysinfo
return win_sysinfo.collect()
根据不同的系统平台，开始收集数据
sysinfo.py
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
import os,sys,subprocess
# import commands
import re
def collect():
filter_keys = ['Manufacturer','Serial Number','Product Name','UUID','Wake-up Type']
raw_data = {}
for key in filter_keys:
try:
#cmd_res = subprocess.check_output("sudo dmidecode -t system|grep '%s'" %key,shell=True)
cmd_res = commands.getoutput("sudo dmidecode -t system|grep '%s'" %key)
cmd_res = cmd_res.strip()
res_to_list = cmd_res.split(':')
if len(res_to_list)> 1:#the second one is wanted string
raw_data[key] = res_to_list[1].strip()
else:
raw_data[key] = -1
except Exception as e:
print(e)
raw_data[key] = -2 #means cmd went wrong
data = {"asset_type":'server'}
data['manufactory'] = raw_data['Manufacturer']
data['sn'] = raw_data['Serial Number']
data['model'] = raw_data['Product Name']
data['uuid'] = raw_data['UUID']
data['wake_up_type'] = raw_data['Wake-up Type']
data.update(cpuinfo())
data.update(osinfo())
data.update(raminfo())
data.update(nicinfo())
data.update(diskinfo())
return data
def diskinfo():
obj = DiskPlugin()
return obj.linux()
def nicinfo():
#tmp_f = file('/tmp/bonding_nic').read()
#raw_data= subprocess.check_output("ifconfig -a",shell=True)
raw_data = commands.getoutput("ifconfig -a")
raw_data= raw_data.split("\n")
nic_dic = {}
next_ip_line = False
last_mac_addr = None
for line in raw_data:
if next_ip_line:
#print last_mac_addr
#print line #, last_mac_addr.strip()
next_ip_line = False
nic_name = last_mac_addr.split()[0]
mac_addr = last_mac_addr.split("HWaddr")[1].strip()
raw_ip_addr = line.split("inet addr:")
raw_bcast = line.split("Bcast:")
raw_netmask = line.split("Mask:")
if len(raw_ip_addr) > 1: #has addr
ip_addr = raw_ip_addr[1].split()[0]
network = raw_bcast[1].split()[0]
netmask =raw_netmask[1].split()[0]
#print(ip_addr,network,netmask)
else:
ip_addr = None
network = None
netmask = None
if mac_addr not in nic_dic:
nic_dic[mac_addr] = {'name': nic_name,
'macaddress': mac_addr,
'netmask': netmask,
'network': network,
'bonding': 0,
'model': 'unknown',
'ipaddress': ip_addr,
}
else: #mac already exist , must be boding address
if '%s_bonding_addr' %(mac_addr) not in nic_dic:
random_mac_addr = '%s_bonding_addr' %(mac_addr)
else:
random_mac_addr = '%s_bonding_addr2' %(mac_addr)
nic_dic[random_mac_addr] = {'name': nic_name,
'macaddress':random_mac_addr,
'netmask': netmask,
'network': network,
'bonding': 1,
'model': 'unknown',
'ipaddress': ip_addr,
}
if "HWaddr" in line:
#print line
next_ip_line = True
last_mac_addr = line
nic_list= []
for k,v in nic_dic.items():
nic_list.append(v)
return {'nic':nic_list}
def raminfo():
#raw_data = subprocess.check_output(["sudo", "dmidecode" ,"-t", "17"])
raw_data = commands.getoutput("sudo dmidecode -t 17")
raw_list = raw_data.split("\n")
raw_ram_list = []
item_list = []
for line in raw_list:
if line.startswith("Memory Device"):
raw_ram_list.append(item_list)
item_list =[]
else:
item_list.append(line.strip())
ram_list = []
for item in raw_ram_list:
item_ram_size = 0
ram_item_to_dic = {}
for i in item:
#print i
data = i.split(":")
if len(data) ==2:
key,v = data
if key == 'Size':
#print key ,v
if v.strip() != "No Module Installed":
ram_item_to_dic['capacity'] = v.split()[0].strip() #e.g split "1024 MB"
item_ram_size = int(v.split()[0])
#print item_ram_size
else:
ram_item_to_dic['capacity'] = 0
if key == 'Type':
ram_item_to_dic['model'] = v.strip()
if key == 'Manufacturer':
ram_item_to_dic['manufactory'] = v.strip()
if key == 'Serial Number':
ram_item_to_dic['sn'] = v.strip()
if key == 'Asset Tag':
ram_item_to_dic['asset_tag'] = v.strip()
if key == 'Locator':
ram_item_to_dic['slot'] = v.strip()
#if i.startswith("")
if item_ram_size == 0: # empty slot , need to report this
pass
else:
ram_list.append(ram_item_to_dic)
#get total size(mb) of ram as well
#raw_total_size = subprocess.check_output(" cat /proc/meminfo|grep MemTotal ",shell=True).split(":")
raw_total_size = commands.getoutput("cat /proc/meminfo|grep MemTotal ").split(":")
ram_data = {'ram':ram_list}
if len(raw_total_size) == 2:#correct
total_mb_size = int(raw_total_size[1].split()[0]) / 1024
ram_data['ram_size'] = total_mb_size
#print(ram_data)
return ram_data
def osinfo():
#distributor = subprocess.check_output(" lsb_release -a|grep 'Distributor ID'",shell=True).split(":")
distributor = commands.getoutput(" lsb_release -a|grep 'Distributor ID'").split(":")
#release = subprocess.check_output(" lsb_release -a|grep Description",shell=True).split(":")
release = commands.getoutput(" lsb_release -a|grep Description").split(":")
data_dic ={
"os_distribution": distributor[1].strip() if len(distributor)>1 else None,
"os_release":release[1].strip() if len(release)>1 else None,
"os_type": "linux",
}
#print(data_dic)
return data_dic
def cpuinfo():
base_cmd = 'cat /proc/cpuinfo'
raw_data = {
'cpu_model' : "%s |grep 'model name' |head -1 " % base_cmd,
'cpu_count' : "%s |grep 'processor'|wc -l " % base_cmd,
'cpu_core_count' : "%s |grep 'cpu cores' |awk -F: '{SUM +=$2} END {print SUM}'" % base_cmd,
}
for k,cmd in raw_data.items():
try:
#cmd_res = subprocess.check_output(cmd,shell=True)
cmd_res = commands.getoutput(cmd)
raw_data[k] = cmd_res.strip()
#except Exception,e:
except ValueError as e:
print(e)
data = {
"cpu_count" : raw_data["cpu_count"],
"cpu_core_count": raw_data["cpu_core_count"]
}
cpu_model = raw_data["cpu_model"].split(":")
if len(cpu_model) >1:
data["cpu_model"] = cpu_model[1].strip()
else:
data["cpu_model"] = -1
return data
class DiskPlugin(object):
def linux(self):
result = {'physical_disk_driver':[]}
try:
script_path = os.path.dirname(os.path.abspath(__file__))
shell_command = "sudo %s/MegaCli -PDList -aALL" % script_path
output = commands.getstatusoutput(shell_command)
result['physical_disk_driver'] = self.parse(output[1])
except Exception as e:
result['error'] = e
return result
def parse(self,content):
'''
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
'''
response = []
result = []
for row_line in content.split("\n\n\n\n"):
result.append(row_line)
for item in result:
temp_dict = {}
for row in item.split('\n'):
if not row.strip():
continue
if len(row.split(':')) != 2:
continue
key,value = row.split(':')
name =self.mega_patter_match(key);
if name:
if key == 'Raw Size':
raw_size = re.search('(\d+\.\d+)',value.strip())
if raw_size:
temp_dict[name] = raw_size.group()
else:
raw_size = '0'
else:
temp_dict[name] = value.strip()
if temp_dict:
response.append(temp_dict)
return response
def mega_patter_match(self,needle):
grep_pattern = {'Slot':'slot', 'Raw Size':'capacity', 'Inquiry':'model', 'PD Type':'iface_type'}
for key,value in grep_pattern.items():
if needle.startswith(key):
return value
return False
if __name__=="__main__":
print(DiskPlugin().linux())
sysinfo.py
#_*_coding:utf-8_*_
__author__ = 'Alex Li'
import platform
import win32com
import wmi
import os
def collect():
data = {
'os_type': platform.system(),
'os_release':"%s %s %s "%( platform.release() ,platform.architecture()[0],platform.version()),
'os_distribution': 'Microsoft',
'asset_type':'server',
#'ram':[]
}
#data.update(cpuinfo())
win32obj = Win32Info()
data.update(win32obj.get_cpu_info())
data.update(win32obj.get_ram_info())
data.update(win32obj.get_server_info())
data.update(win32obj.get_disk_info())
data.update(win32obj.get_nic_info())
#for k,v in data.items():
# print k,v
return data
class Win32Info(object):
def __init__(self):
self.wmi_obj = wmi.WMI()
self.wmi_service_obj = win32com.client.Dispatch("WbemScripting.SWbemLocator")
self.wmi_service_connector =self.wmi_service_obj.ConnectServer(".","root\cimv2")
def get_cpu_info(self):
data = {}
cpu_lists = self.wmi_obj.Win32_Processor()
cpu_core_count = 0
for cpu in cpu_lists:
cpu_core_count += cpu.NumberOfCores
cpu_model = cpu.Name
data["cpu_count"] = len(cpu_lists)
data["cpu_model"] = cpu_model
data["cpu_core_count"] =cpu_core_count
return data
def get_ram_info(self):
data = []
ram_collections = self.wmi_service_connector.ExecQuery("Select * from Win32_PhysicalMemory")
for item in ram_collections:
item_data = {}
#print item
mb = int(1024 * 1024)
ram_size = int(item.Capacity) / mb
item_data = {
"slot":item.DeviceLocator.strip(),
"capacity":ram_size,
"model":item.Caption,
"manufactory":item.Manufacturer,
"sn":item.SerialNumber,
}
data.append(item_data)
#for i in data:
# print i
return {"ram":data}
def get_server_info(self):
computer_info = self.wmi_obj.Win32_ComputerSystem()[0]
system_info = self.wmi_obj.Win32_OperatingSystem()[0]
data = {}
data['manufactory'] = computer_info.Manufacturer
data['model'] = computer_info.Model
data['wake_up_type'] = computer_info.WakeUpType
data['sn'] = system_info.SerialNumber
#print data
return data
def get_disk_info(self):
data = []
for disk in self.wmi_obj.Win32_DiskDrive():
#print disk.Model,disk.Size,disk.DeviceID,disk.Name,disk.Index,disk.SerialNumber,disk.SystemName,disk.Description
item_data = {}
iface_choices = ["SAS","SCSI","SATA","SSD"]
for iface in iface_choices:
if iface in disk.Model:
item_data['iface_type'] = iface
break
else:
item_data['iface_type'] = 'unknown'
item_data['slot'] = disk.Index
item_data['sn'] = disk.SerialNumber
item_data['model'] = disk.Model
item_data['manufactory'] = disk.Manufacturer
item_data['capacity'] = int(disk.Size ) / (1024*1024*1024)
data.append(item_data)
return {'physical_disk_driver':data}
def get_nic_info(self):
data = []
for nic in self.wmi_obj.Win32_NetworkAdapterConfiguration():
if nic.MACAddress is not None:
item_data = {}
item_data['macaddress'] = nic.MACAddress
item_data['model'] = nic.Caption
item_data['name'] = nic.Index
if nic.IPAddress is not None:
item_data['ipaddress'] = nic.IPAddress[0]
item_data['netmask'] = nic.IPSubnet
else:
item_data['ipaddress'] = ''
item_data['netmask'] = ''
bonding = 0
#print nic.MACAddress ,nic.IPAddress,nic.ServiceName,nic.Caption,nic.IPSubnet
#print item_data
data.append(item_data)
return {'nic':data}
if __name__=="__main__":
collect()
待存区：临时存储完成资产信息审批过滤
class NewAssetApprovalZone(models.Model):
sn = models.CharField(u'资产SN号',max_length=128, unique=True)
asset_type_choices = (
('server', u'服务器'),
('switch', u'交换机'),
('router', u'路由器'),
('firewall', u'防火墙'),
('storage', u'存储设备'),
('NLB', u'NetScaler'),
('wireless', u'无线AP'),
('software', u'软件资产'),
('others', u'其它类'),
)
asset_type = models.CharField(choices=asset_type_choices,max_length=64,blank=True,null=True)
manufactory = models.CharField(max_length=64,blank=True,null=True)
model = models.CharField(max_length=128,blank=True,null=True)
ram_size = models.IntegerField(blank=True,null=True)
cpu_model = models.CharField(max_length=128,blank=True,null=True)
cpu_count = models.IntegerField(blank=True,null=True)
cpu_core_count = models.IntegerField(blank=True,null=True)
os_distribution = models.CharField(max_length=64,blank=True,null=True)
os_type = models.CharField(max_length=64,blank=True,null=True)
os_release = models.CharField(max_length=64,blank=True,null=True)
data = models.TextField(u'资产数据')
date = models.DateTimeField(u'汇报日期',auto_now_add=True)
approved = models.BooleanField(u'已批准',default=False)
approved_by = models.ForeignKey('UserProfile',verbose_name=u'批准人',blank=True,null=True)
approved_date = models.DateTimeField(u'批准日期',blank=True,null=True)
def __str__(self):
return self.sn
class Meta:
verbose_name = '新上线待批准资产'
verbose_name_plural = "新上线待批准资产"
客户端agent发送资产信息到服务端
发送流程：
客户端第一次发送资产信息时，存入暂存数据表中，审核通过后再存入正式资产数据表中，
存入后服务端返回一个id给到客户端，并保存在本地文件中，客户端第二次发送资产信息时，将会携带该id，完成资产数据表的更新。
正式发送资产信息
def report_asset(self):
obj = info_collection.InfoCollection()
asset_data = obj.collect()
asset_id = self.load_asset_id(asset_data["sn"])
if asset_id: #reported to server before
asset_data["asset_id"] = asset_id
post_url = "asset_report"
else:#first time report to server
'''report to another url,this will put the asset into approval waiting zone, when the asset is approved ,this request returns
asset's ID'''
asset_data["asset_id"] = None
post_url = "asset_report_with_no_id"
data = {"asset_data": json.dumps(asset_data)}
response = self.__submit_data(post_url,data,method="post")
if "asset_id" in response:
self.__update_asset_id(response["asset_id"])
self.log_record(response)
根据不同的请求方式get/post，选择不同的处理，完成组装Url及对应的参数发送数据
def __submit_data(self,action_type,data,method):
'''
send data to server
:param action_type: url
:param data: 具体要发送的数据
:param method: get/post
:return:
'''
if action_type in settings.Params['urls']:
if type(settings.Params['port']) is int:
url = "http://%s:%s%s" %(settings.Params['server'],settings.Params['port'],settings.Params['urls'][action_type])
else:
url = "http://%s%s" %(settings.Params['server'],settings.Params['urls'][action_type])
#url = self.__attach_token(url)
print('Connecting [%s], it may take a minute' % url)
if method == "get":
args = ""
for k,v in data.items():
args += "&%s=%s" %(k,v)
args = args[1:]
url_with_args = "%s?%s" %(url,args)
print(url_with_args)
try:
req = urllib.request.urlopen(url_with_args,timeout=settings.Params['request_timeout'])
#req_data =urlopen(req,timeout=settings.Params['request_timeout'])
#callback = req_data.read()
callback = req.read()
print("-->server response:",callback)
return callback
except urllib.URLError as e:
sys.exit("\033[31;1m%s\033[0m"%e)
elif method == "post":
try:
data_encode = urllib.parse.urlencode(data).encode()
req = urllib.request.urlopen(url=url,data=data_encode,timeout=settings.Params['request_timeout'])
#res_data = urllib.urlopen(req,timeout=settings.Params['request_timeout'])
callback = req.read()
callback = json.loads(callback.decode())
print("\033[31;1m[%s]:[%s]\033[0m response:\n%s" %(method,url,callback) )
return callback
except Exception as e:
sys.exit("\033[31;1m%s\033[0m"%e)
else:
raise KeyError