python解析jmx完成jmeter压力测试

最近在在弄项目的测试质量平台，平台本身是django+vue搭建的，缺少 一个性能压测的小模块，所以着手处理了了下，其实第一选择应该 是locust，因为locust本身就是基于python的性能测试 框架，可能跟平台本身更加契合，但在实际的测试和体验后，发现其实使用并不理想，占用内存大，数据指标也并不准确。后来觉得还是用jmeter处理下可能更稳妥些。

大体的思路如下 ：

1、点击平台内case列表执行按键，将case相关信息如（线程数，持续时间，请求 方式、参数等）发送 至mq

2、压力机内mq进行消费 读取case信息，将信息写入jmeter的JMX文件内，执行该JMX文件

3、生成结果后，因为jmeter生成报告的方式是内部引用变量，无法从某个节点取到对应数值，所以要将生成的jtl文件转为csv文件

4、最后将csv文件写入数据库，平台列表展示case关联压测结果

import time, pika, os, json, subprocess, csv, pymysql, logging
import xml.etree.cElementTree as ET
from urllib import parse

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"  # 日志格式化输出
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"  # 日期格式
fp = logging.FileHandler('performance.log', encoding='utf-8')
fs = logging.StreamHandler()
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT, handlers=[fp, fs])


# jmeter  基本信息
currpath = os.path.dirname(os.path.realpath(__file__))

JMETER_PLUGIN_NAME = r'''"E:\自动化\jmeter\apache-jmeter-3.1\apache-jmeter-3.1\lib\ext\CMDRunner.jar"'''

JMETER_HOME = r'''"E:\自动化\jmeter\apache-jmeter-3.1\apache-jmeter-3.1\bin\jmeter.bat"'''

JMETER_EXEC_JMX = "excutePerformance.jmx"

JMETER_TEMPLATES_JMX = "templatesPerformance.jmx"

# rabbitmq 配置信息
MQ_CONFIG = {
    "host": "",
    "port": 5672,
    "vhost": "/",
    "user": "admin",
    "passwd": "admin",
}


# mq 消费端
def consume():
    # 创建连接
    credentials = pika.PlainCredentials(MQ_CONFIG["user"], MQ_CONFIG["passwd"])
    conn_broker = pika.BlockingConnection(
        pika.ConnectionParameters(host=MQ_CONFIG["host"], port=MQ_CONFIG["port"], virtual_host=MQ_CONFIG["vhost"],
                                  credentials=credentials))

    # 在连接上创建一个频道
    chan = conn_broker.channel()
    chan.queue_declare(queue='performanceQueue')
    chan.basic_consume('performanceQueue', callback, auto_ack=True)
    chan.start_consuming()


# 解析执行mq数据
def callback(ch, method, properties, body):
    strBody = body.decode('gbk').replace("'", '"')
    requestDic = json.loads(strBody)
    if handleTemplates(requestDic):
        execjmxs(requestDic["case_per_result_id"])


# 将mq信息写入jmx文件
def handleTemplates(requestDic):
    try:
        logging.info("将mq信息写入jmx模板")
        # 基本信息
        url = parse.urlparse(requestDic["request"]["host"])
        HTTPSampler_domain = url.netloc
        HTTPSampler_protocol = url.scheme
        HTTPSampler_path = requestDic["request"]["path"]
        HTTPSampler_method = requestDic["request"]["method"]
        HTTPSampler_params = requestDic["request"]["params"]
        HTTPSampler_body = requestDic["request"]["data"]
        HTTPSampler_headers = requestDic["request"]["header"]
        HTTPSampler_expect = requestDic["request"]["expect"]

        # 线程组设置
        ThreadGroup_num_threads = str(requestDic["threadCount"])
        ThreadGroup_num_ramp_time = "20"
        ThreadGroup_num_duration = str(requestDic["timelong"])
        # consume()
        tree = ET.parse(JMETER_TEMPLATES_JMX)
        root = tree.getroot()

        # 处理threadCount
        ThreadGroup = tree.find("hashTree/hashTree/ThreadGroup")
        stringPropList = ThreadGroup.iter('stringProp')
        for stringProp in stringPropList:
            if stringProp.attrib["name"] == "ThreadGroup.num_threads":
                stringProp.text = ThreadGroup_num_threads
            if stringProp.attrib["name"] == "ThreadGroup.ramp_time":
                stringProp.text = ThreadGroup_num_ramp_time
            if stringProp.attrib["name"] == "ThreadGroup.duration":
                stringProp.text = ThreadGroup_num_duration

        # 处理请求头 "hashTree/hashTree/hashTree/HeaderManager"
        HeaderManager = tree.find("hashTree/hashTree/hashTree/HeaderManager")
        collectionProp = ET.Element("collectionProp", {"name": "HeaderManager.headers"})
        if HTTPSampler_headers:
            for item in HTTPSampler_headers:
                elementProp = ET.Element("elementProp", {"name": "", "elementType": "Header"})
                stringPropName = ET.Element("stringProp", {"name": "Header.name"})
                stringPropName.text = item["key"]
                elementProp.append(stringPropName)
                stringPropValue = ET.Element("stringProp", {"name": "Header.value"})
                stringPropValue.text = item["value"]
                elementProp.append(stringPropValue)
                collectionProp.append(elementProp)
        HeaderManager.append(collectionProp)

        # 处理请求 基本信息   host  协议   路径  等
        HTTPSamplerProxy = tree.find("hashTree/hashTree/hashTree/HTTPSamplerProxy")
        # for HTTPSamplerProxy in root.iter('HTTPSamplerProxy'):  # element.findall()查询当前元素的子元素
        stringPropList = HTTPSamplerProxy.iter('stringProp')
        for stringProp in stringPropList:
            if stringProp.attrib["name"] == "HTTPSampler.domain":
                stringProp.text = HTTPSampler_domain
            if stringProp.attrib["name"] == "HTTPSampler.protocol":
                stringProp.text = HTTPSampler_protocol
            if stringProp.attrib["name"] == "HTTPSampler.path":
                stringProp.text = HTTPSampler_path
            if stringProp.attrib["name"] == "HTTPSampler.method":
                stringProp.text = HTTPSampler_method

        if HTTPSampler_method == "GET" and HTTPSampler_params:
            elementProp = ET.Element("elementProp", {"name": "HTTPsampler.Arguments",
                                                     "elementType": "Arguments",
                                                     "guiclass": "HTTPArgumentsPanel",
                                                     "testclass": "Arguments",
                                                     "testname": "用户定义的变量",
                                                     "enabled": "true"})
            collectionProp = ET.Element("collectionProp", {"name": "Arguments.arguments"})
            for param in HTTPSampler_params:
                elementProp_param = ET.Element("elementProp", {"name": param["key"], "elementType": "HTTPArgument"})
                boolProp = ET.Element("boolProp", {"name": "HTTPArgument.always_encode"})
                boolProp.text = "false"
                elementProp_param.append(boolProp)

                stringProp = ET.Element("stringProp", {"name": "Argument.value"})
                stringProp.text = str(param["value"])
                elementProp_param.append(stringProp)

                stringProp = ET.Element("stringProp", {"name": "Argument.metadata"})
                stringProp.text = "="
                elementProp_param.append(stringProp)

                boolProp = ET.Element("boolProp", {"name": "HTTPArgument.use_equals"})
                boolProp.text = "true"
                elementProp_param.append(boolProp)

                stringProp = ET.Element("stringProp", {"name": "Argument.name"})
                stringProp.text = param["key"]
                elementProp_param.append(stringProp)

                collectionProp.append(elementProp_param)

            elementProp.append(collectionProp)
            HTTPSamplerProxy.append(elementProp)

        if HTTPSampler_method == "POST" and HTTPSampler_body:
            boolProp = ET.Element("boolProp", {"name": "HTTPSampler.postBodyRaw"})
            boolProp.text = "true"
            HTTPSamplerProxy.append(boolProp)

            elementProp = ET.Element("elementProp", {"name": "HTTPsampler.Arguments",
                                                     "elementType": "Arguments"})
            collectionProp = ET.Element("collectionProp", {"name": "Arguments.arguments"})

            elementProp_Body = ET.Element("elementProp", {"name": "",
                                                          "elementType": "HTTPArgument"})

            boolProp = ET.Element("boolProp", {"name": "HTTPArgument.always_encode"})
            boolProp.text = "false"
            elementProp_Body.append(boolProp)

            stringProp = ET.Element("stringProp", {"name": "Argument.value"})
            stringProp.text = HTTPSampler_body
            elementProp_Body.append(stringProp)
            print(stringProp.text)

            stringProp = ET.Element("stringProp", {"name": "Argument.metadata"})
            stringProp.text = "="
            elementProp_Body.append(stringProp)

            collectionProp.append(elementProp_Body)
            elementProp.append(collectionProp)
            HTTPSamplerProxy.append(elementProp)

        # 断言
        ResponseAssertion_stringProp = tree.find(
            "hashTree/hashTree/hashTree/ResponseAssertion/collectionProp/stringProp")
        ResponseAssertion_stringProp.text = HTTPSampler_expect

        tree.write(JMETER_EXEC_JMX, encoding='UTF-8')
        print("完成")
        return True
    except Exception as message:
        logging.error("可执行jmx生成失败：%s" % str(message))
        return False


# jmeter脚本初始化

# 执行jmeter
def execcmd(command, filename, case_per_result_id):
    try:
        output = subprocess.Popen(
            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
            universal_newlines=True)
        output.communicate()
        if output.returncode == 0:
            jtlToCsv(filename, case_per_result_id)
    except Exception as message:
        logging.error("执行jmeter压力测试失败:%s" % str(message))


# jtl数据转换至csv文件读取
def jtlToCsv(filename, case_per_result_id):
    try:
        command = f"java -jar {JMETER_PLUGIN_NAME} --tool Reporter  --generate-csv {filename}.csv --input-jtl {filename}.jtl  --plugin-type AggregateReport"

        output = subprocess.Popen(
            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
            universal_newlines=True)

        output.communicate()
        if output.returncode == 0:
            cvsToData(f"{filename}.csv", case_per_result_id)

    except Exception as message:
        logging.error("jtl数据转换至csv文件读取失败：%s" % str(message))


# 将csv文件结果存储至数据库
def cvsToData(filePath, case_per_result_id):
    """

    :param filePath: csv 文件路径，存储为数据结果
    :param case_per_result_id: case运行结果存储表 id
    :return:
    """
    db = pymysql.connect(host='', port=3306, database='testdatabase', user='admin',
                         password='admin', charset='utf8')
    cursor = db.cursor()
    try:
        logging.info("%s :读取数据并插入数据库" % case_per_result_id)
        result_Time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        with open(filePath, 'r') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                if row['sampler_label'] == "总体":
                    sql_command = f" update case_performance_result set " \
                        f"aggregate_report_count=\'{row['aggregate_report_count']}\'," \
                        f" average=\'{row['average']}\'," \
                        f"aggregate_report_median=\'{row['aggregate_report_median']}\'," \
                        f" aggregate_report_90_line=\'{row['aggregate_report_90%_line']}\'," \
                        f" aggregate_report_min=\'{row['aggregate_report_min']}\'," \
                        f"aggregate_report_max=\'{row['aggregate_report_max']}\'," \
                        f" aggregate_report_error=\'{row['aggregate_report_error%']}\'," \
                        f"aggregate_report_rate=\'{row['aggregate_report_rate']}\', " \
                        f"aggregate_report_bandwidth=\'{row['aggregate_report_bandwidth']}\'," \
                        f" aggregate_report_stddev=\'{row['aggregate_report_stddev']}\'," \
                        f"result_date=\'{result_Time}\'," \
                        f"case_per_status=1  where id={case_per_result_id}"
                    # sql_command = f"INSERT INTO case_performance_result VALUES (null ,\'{row['aggregate_report_count']}\', \'{row['average']}\',\'{row['aggregate_report_median']}\', \'{row['aggregate_report_90%_line']}\', \'{row['aggregate_report_min']}\',\'{row['aggregate_report_max']}\', \'{row['aggregate_report_error%']}\',\'{row['aggregate_report_rate']}\', \'{row['aggregate_report_bandwidth']}\', \'{row['aggregate_report_stddev']}\',\'{ceateTime}\',{caseId})"
                    print(sql_command)
                    cursor.execute(sql_command)
                    db.commit()
                    db.rollback()
                    logging.info("%s:数据插入成功" % case_per_result_id)


    except Exception as message:
        logging.error(str(message))
    db.close()


# 执行jmter命令
def execjmxs(case_per_result_id):
    try:
        tmpData = ''
        with open(JMETER_EXEC_JMX, "r", encoding="utf-8") as file:
            tmpData = file.read()
        now = time.strftime(r'%Y%m%d%H%M%S', time.localtime(time.time()))
        tmpjmxfile = currpath + r"/{0}_{1}.jmx".format(
            case_per_result_id, now)
        with open(tmpjmxfile, "w+", encoding="utf-8") as file:
            file.writelines(tmpData)
        filename = currpath + "/result_{0}_{1}".format(case_per_result_id, now)
        commond = f"{JMETER_HOME} -n -t {tmpjmxfile} -l {filename}.jtl"
        execcmd(commond, filename, case_per_result_id)
    except Exception as message:
        logging.error("执行jmter命令失败:%s" % str(message))


if __name__ == '__main__':
    consume()

大体的脚本思路如下 ，其实 就是 将我们的一些接口基本信息和执行参数写入jmx文件中，文中为调试的demo没有 仔细雕琢，其实 可能有更好的解决方式，但感觉改下jmx是比较快速快速的落地方案，后续可以扩展一些自己只需要的功能，多集群的压力测试，监控被压测机的物理主机状态等，这里大概就是一些简答思路 。