SQLServer 跟踪语句转为Jmeter脚本
待测系统为CS架构，后台为SQLserver数据库，目前针对CS架构没有有效的压力测试工具，只能通过抓取业务SQL，压数据库的方式进行测试。因为涉及到的SQL语句比较多，所以通过SQL监控工具，获取到SQL语句，然后转为Jmeter脚本（xml）的形式。脚本分为量大部分：解析抓取到的SQL、将解析的SQL转为Jmeter脚本。多个跟踪文件，在同一个文件夹中，通过轮询方式，转为对应的线程组。
解析跟踪文件
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2019-12-24 13:29
# @Author : 无名氏
# @File : resolvsqlxml.py
import warnings
warnings.filterwarnings("ignore") #忽略warning信息
from xml.dom.minidom import parse
from datetime import datetime
import xml.dom.minidom
# import string
import os,time,re
import codecs
import chardet
import pandas as pd
class ResolveXmlfile():
def __init__(self,xmlfile,directory):
self.xmlfile = xmlfile
self.xml_directory = directory
self.df = pd.DataFrame(columns=('sys_name','db', 'time', 'sqltext', 'timestamp')) #创建DF,定义列名
self.exclude_sql_list = ['SP:Starting', 'Trace Stop', 'Trace Start', 'Trace Pause'] #筛选掉指定类型的语句
def get_xml_file(self):
'''
判断是否有传递文件，True 只处理传递的文件，false 处理文件夹下的所有文件
'''
if self.xmlfile:
print(self.xmlfile)
file_path = os.path.join(self.xml_directory, self.xmlfile)
(file_name, extension) = os.path.splitext(self.xmlfile)
print(file_path, file_name)
self.reslove_xmlfile(file_path, file_name)
else:
for dirpath, dirnames, files in os.walk(self.xml_directory):
for file in files:
file_path = os.path.join(dirpath, file)
(file_name, extension) = os.path.splitext(file)
# print(file_path,file_name)
self.reslove_xmlfile(file_path,file_name)
# sqldf = self.df.sort_values(by="timestamp", axis=0, ascending=True, inplace=False) # 按照时间戳排序
return self.df
def get_xmlnode(self,node, name): # 获取xml节点
return node.getElementsByTagName(name) if node else []
def get_nodevalue(self,node, index=0): #获取节点Value---01 01
return node.getAttribute(attrname) if node else ''
def reslove_xmlfile(self,file_path,file_name):
try:
dom_tree = xml.dom.minidom.parse(file_path)
dom = dom_tree.documentElement
event_node_list = self.get_xmlnode(dom, 'Event')
for event_node in event_node_list:
# print('==========')
event_name = self.get_attrvalue(event_node,'name')
if event_name and event_name not in self.exclude_sql_list:
column_node_list = self.get_xmlnode(event_node, 'Column')
db_value, app_value, sql_value, startTime_value = '', '', '', '' #给需要抓取的字段赋初始值空值
column_dict = {}
for column_node in column_node_list:
column_name = self.get_attrvalue(column_node, 'name')
if column_name in ('DatabaseName','ApplicationName','TextData','StartTime'):
column_dict[column_name] = self.get_nodevalue(column_node)
if "TextData" in column_dict.keys():
# print('column_dict',column_dict.keys())
db_value = column_dict.get("DatabaseName", '')
sql_value = column_dict.get("TextData", '')
app_value = column_dict.get("ApplicationName", '')
startTime_value = column_dict.get("StartTime", '')
if startTime_value:
timestamp_value = self.timestamp_conversion(startTime_value)
else:
timestamp_value=''
# print('sql:',sql_value)
#筛选异常数据 'sys_name','db', 'time', 'sqltext', 'timestamp'
if (startTime_value != '') and (db_value not in ('msdb', 'master')) and ('SQLAgent' not in app_value) and (sql_value != 'exec sp_reset_connection'):
self.df = self.df.append(pd.DataFrame({
'sys_name': [file_name],
'sqltext': [sql_value],
'db': [db_value],
'time': [startTime_value],
'timestamp': [timestamp_value]
}))
except Exception as e:
print(e)
def timestamp_conversion(self,time_value):
'''
time.time() 生成当前的时间戳，格式为10位整数的浮点数。
time.strftime()根据时间元组生成时间格式化字符串。
time.strptime()根据时间格式化字符串生成时间元组。time.strptime()与time.strftime()为互操作。
timearray = time.strptime(time1, "%Y-%m-%d %H:%M:%S.%f")
time.localtime()根据时间戳生成当前时区的时间元组。
time.mktime()根据时间元组生成时间戳。
time.mktime(timearray)
'''
#抓取的xml数据 时间格式为2020-03-27T11:21:05.163+08:00 去除+8:00和T
time_value = time_value.split('+')[0].replace('T',' ')
if '.' in time_value: #判断时间是否有毫秒数据
time_array = datetime.strptime(time_value, "%Y-%m-%d %H:%M:%S.%f") #
millisecond_stamp = int(time_array.microsecond/1000)
timestamp= int(time.mktime(time_array.timetuple())*1000)
timestamp = millisecond_stamp+timestamp
else:
time_array = datetime.strptime(time_value, "%Y-%m-%d %H:%M:%S")
timestamp = int(time.mktime(time_array.timetuple())*1000)
return timestamp
def PrintSQl(self,sqldf):
j = 1
print(sqldf.iterrows())
for index,row in sqldf.iterrows(): #需要优化
print('\n\n>>>>>>>>>>>>>>>', j, '<<<<<<<<<<<<<<<<<<<')
print('sys_name :', row['sys_name'])
# print('STARTIME :', row['time'])
# print('DBNAME :', row['dbname'])
# print('-------------', j, '-------------')
# print('SQLTEXT :', row['sqltext'])
j = j + 1
print('数据库跟踪xml文件解析完成')
def group_df(self,sqldf):
# print(sqldf.groupby('sys_name').groups.keys())
# for file in sqldf.groupby('sys_name').groups.keys():
# print(file)
# sqldf.groupby('sys_name')
for groupname,grouplist in sqldf.groupby('sys_name'):
print(groupname)
print(type(grouplist))
for index, row in grouplist.iterrows():
print(row['sqltext'])
xmlfile = ['']
filepath = 'XMlFile/'
exclude_sql_list = ['SP:Starting', 'Trace Stop', 'Trace Start', 'Trace Pause']
if __name__ == "__main__":
run = ResolveXmlfile(xmlfile,filepath,exclude_sql_list)
# run.handlxml()
sql_pd = run.get_xml_file()
run.group_df(sql_pd)
# run.test()
# run.PrintSQl(sql_pd)
# run.timestamp_conversion()
转Jmeter脚本
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2019-12-20 20:52
# @Author : 无名氏
# @File : WriteJmxFile.py
import xml.dom.minidom as Dom
from xml.dom import minidom
import random
import time,json
from Fun.GetConf import *
import pandas as pd
from resolvsqlxml import *
import chardet
import re
# from Fun.logger import *
class SqlToJmeterml():
def __init__(self,jmxfile,jmxpath,confpath,logger):
self.dom = minidom.Document()
self.hashtree_lev1 = self.dom.createElement('hashTree') # 创建节点hashTree ,父节点jmeternode
self.hashtree_lev2 = self.dom.createElement('hashTree')
self.hashTree_lev3 = self.dom.createElement('hashTree')
self.none_value = self.dom.createTextNode('')
self.true_value = self.dom.createTextNode('true')
self.false_value = self.dom.createTextNode('false')
self.lastnode = ''
self.jmxfile = jmxfile
self.path = confpath
self.logger= logger
self.jmxpath = jmxpath
def get_node_confinfo(self,node,elementProp):
attr_list = json.loads(GetConfigure(self.path, node, 'Itemlist'))
node_list = json.loads(GetConfigure(self.path, node, 'Nodelist'))
if elementProp:
elementProp_attr_list = json.loads(GetConfigure(self.path, node, 'elementPropItemlist'))
elementProp_node_list = json.loads(GetConfigure(self.path, node, 'elementPropNodelist'))
return attr_list, node_list, elementProp_attr_list, elementProp_node_list
else:
return attr_list, node_list
def jmeterTestPlan(self):
# 添加JmeterTestPlan节点--根节点
jmeterTestPlan_attr_list = [{'version': '1.2'}, {'properties': '5.0'}, {'jmeter': '5.2'}]
jmeterTestPlan_Node = self.AddNode('jmeterTestPlan', jmeterTestPlan_attr_list, None)
self.dom.appendChild(jmeterTestPlan_Node)
jmeterTestPlan_Node.appendChild(self.hashtree_lev1)
def Testplan(self):
try:
TestPlan_attr,TestPlan_node,TestPlan_elementProp_attr,TestPlan_elementProp_node = self.get_node_confinfo('TestPlan','elementProp')
# 创建节点，
TestPlan_elementProp = self.AddNode('elementProp', TestPlan_elementProp_attr, TestPlan_elementProp_node)
TestPlan = self.AddNode('TestPlan', TestPlan_attr, TestPlan_node)
# 将节点添加到父节点中
TestPlan.appendChild(TestPlan_elementProp)
self.hashtree_lev1.appendChild(TestPlan)
self.hashtree_lev1.appendChild(self.hashtree_lev2)
print('TestPlan节点创建完毕')
except Exception as e:
print(e)
def JDBCDataSource(self):
try:
#获取需要配置的数据库
db_sections = GetConfigure(self.path, 'database','')
for db_section in db_sections:
db = GetConfigure(self.path, 'database',db_section)
url = GetConfigure(self.path, db, 'url')
dbname = GetConfigure(self.path, db, 'name')
username = GetConfigure(self.path, db, 'username')
password = GetConfigure(self.path, db, 'password')
dburl = 'jdbc:sqlserver://' + url + ';databaseName=' + dbname + ';'
jdbc_testname = dict(testname=dbname) #定义数据库连接名称
jdbc_attr,jdbc_node = self.get_node_confinfo('JDBC', '')
jdbc_attr.append(jdbc_testname)
for node in jdbc_node:
(key, value), = node.items()
if isinstance(value, list):
if value[0] == 'dataSource':
value.append(dbname)
elif value[0] == 'dbUrl':
value.append(dburl)
elif value[0] == 'password':
value.append(password)
elif value[0] == 'username':
value.append(username)
#创建JDBC节点
jdbcdatasource = self.AddNode('JDBCDataSource', jdbc_attr, jdbc_node)
self.hashtree_lev2.appendChild(jdbcdatasource)
#添加每个节点之间的目录结构
