压力测试监控数据通过python matplotlib进行可视化

执行压力测试过程中，监控服务器性能变化，通过常用的命令可以观察当前时刻性能情况，但是无法判断整个测试过程中性能的变化趋势。
随后，将压力测试过程中服务器性能指标按照10s一次进行采集并写入文件中。完成测试后，通过脚本解析采集的性能数据，使用python的三方模块matplotlib对指标进行绘图。

不足之处是，目前这个脚本只能在压力测试结束后进行曲线图绘制，而不能在压力测试过程中实施绘制

指标文件数据格式如下：

Network_Flow
In_link Out_link
Linking_Number Net_Traffic(MB)
1.87925 0.119033
4.5772 0.141904
2.62263 0.126162
2.50861 0.12293
4.92332 0.139697
3.24791 0.126943

解析指标文件的代码如下：

# coding:utf-8
__author__ = 'Libiao'
import os,sys
curr_dir = os.getcwd()

def is_multi_lines(linelabel):
    if ' ' in linelabel.strip():
        return True
    else:
        return False

def read_datas(args=()):
    dir_name,files_dict = args
    result = {}
    for groupname,files_list in files_dict.items():
        res = []
        res_dir = curr_dir + "\\" + dir_name + "\\"
        if not os.listdir(res_dir):
            continue
        for files in files_list:
            try:
                with file(res_dir + files,'r') as f:
                    datas = [data for data in f]
                    title,line_labels,x_y_labels = datas.pop(0),datas.pop(0),datas.pop(0)
                    x_label,y_label = x_y_labels.strip("\n").split(' ')
                    if is_multi_lines(line_labels):
                        line_label_first,line_label_second = line_labels.strip("\n").split(" ")
                        line_data_first = [d.strip("\n").split(' ')[0] for d in datas]
                        line_data_second = [d.strip("\n").split(' ')[1] for d in datas if d]
                        res.append((title,line_label_first,line_label_second,x_label,y_label,line_data_first,line_data_second))
                    else:
                        line_data = [d.strip() for d in datas]
                        res.append((title,line_labels,x_label,y_label,line_data))
            except IOError as ioerr:
                print str(ioerr)
        result[groupname] = res
    return (dir_name,result)

解析配置文件的代码如下：

# coding:utf-8
__author__ = 'Libiao'

import os
from xml.etree import ElementTree as ET

def read_xml(xmlpath):
    res = []
    root = ET.parse(xmlpath)
    rootnode_list = root.getiterator("dir")
    for node in rootnode_list:
        files_map = {}
        dir_name = node.attrib["name"]
        for second_node in node.getiterator("group"):
            group_name = second_node.attrib['name']
            tmp = []
            for child in second_node.getchildren():
                tmp.append(child.text)
            files_map[group_name] = tmp
        res.append((dir_name,files_map))
    return res

进行曲线图绘制的代码：

#coding:utf-8

import matplotlib.pyplot as plt
from pylab import *
import numpy as np
import random
import read_kpi_data,xml_parser
import threading,subprocess,multiprocessing
import os

curr_dir = os.getcwd()

color = ['r','g','b','y','c','k']

#获取测试时间值
def get_test_time(dir_name):
    with file(curr_dir + "\\" + dir_name + "\\timeConsum",'r') as f:
        times = int(float(f.readline().strip()))
    return times

#获取服务连接数
def get_server_num(dir_name):
    ser_num = []
    try:
        with file(curr_dir + "\\" + dir_name + "\\linking_number","r") as f:
            for value in f:
                if value:
                    ser_num.append(int(value))
    except Exception as e:
        print str(e)
    return ser_num

#获取图表布局rows和cols
def get_layout(graph_num):
    rows,cols = None,None
    if graph_num == 0:
        return None
    elif graph_num == 1:
        rows,cols = 1,1
    elif graph_num < 4 and graph_num not in (0,1):
        rows,cols = graph_num,1
    elif graph_num >= 4 and graph_num <= 10:
        if graph_num % 2 == 0:
            rows = graph_num / 2
        else:
            rows = graph_num / 2 + 1
        cols = 2
    else:
        if graph_num % 3 == 0:
            rows = graph_num / 3
        else:
            rows = graph_num / 3 + 1
        cols = 3
    return (rows,cols)

def graph_display(plot_name,all_datas):
    plt.figure(plot_name,figsize=(10,8),dpi=80)
    #设置子图在画板中的布局，主要是子图高宽（hspace、wspace）、上下边距（bottom、top），左右边距（left、right）
    # plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,wspace=0.25,hspace=0.45)
    if len(all_datas) > 10:
        plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,hspace=0.75)
    elif len(all_datas)>6 and len(all_datas) <= 10:
        plt.subplots_adjust(bottom=0.08,top=0.95,hspace=0.35)
    #从time_consum文件读取性能指标采集消耗的时间
    #x_axle = get_test_time(plot_name)
    #从linking_number文件读取连接数
    x_axle = get_server_num(plot_name)
    #获取图表布局
    rows,cols = get_layout(len(all_datas))
    colorLen = len(color)
    for index, datas in enumerate(all_datas):
        yMax,yMin = 0.0,0.0
        subplt = plt.subplot(rows,cols,index+1)
        title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = None,None,None,None,None,None,None
        try:
            title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = datas
        except:
            title,first_line_label,x_label,y_label,first_line_datas = datas

        tmp = None
        #设置子图标题
        subplt.set_title(title.strip("\n"))
        x = np.linspace(0,np.array(get_server_num(plot_name)).max(),len(first_line_datas))
        #x = np.array(get_server_num(plot_name))
        if second_line_label == None:
            tmp = [float(d) for d in first_line_datas]
            subplt.plot(x,first_line_datas,color = color[index % colorLen],linewidth=1.5,linestyle='-',label=first_line_label.strip("\n"))
        else:
            tmp1 = [float(d) for d in first_line_datas if d]
            tmp2 = [float(d) for d in second_line_datas if d]
            try:
                tmp = [max(tmp1),max(tmp2),min(tmp1),min(tmp2)]
            except ValueError as e:
                print title
            num = -1 #num=-1,是为了防止color = color[index % colorLen + num]计算中color下标越界
            for line_label,line_data in zip([first_line_label,second_line_label],[first_line_datas,second_line_datas]):
                subplt.plot(x,line_data,color = color[index % colorLen + num],linewidth=1.5,linestyle='-',label=line_label.strip("\n"))
                subplt.legend()
                num += 1
        #读取连接数，绘制到每个图表
        with open("SYS_KPI/srs_number",'r') as f:
            datas = f.readlines()
        yMax = max(tmp)
        yMin = min(tmp)
        dx = (x.max() - x.min()) * 0.1
        dy = (yMax - yMin) * 0.1
        subplt.set_ylim(yMin - dy,yMax + dy)
        subplt.set_xlim(x.min() - dx,x.max() + dx)
        #{'position':(1.0,1.0)}，其中1.0相当于x轴最右边，0表示最左边，0.5便是中间
        subplt.set_xlabel(x_label,{'color':'b','position':(1.0,1.0)})
        subplt.set_ylabel(y_label,{'color':'b'})
        #子图显示网格
        subplt.grid(True)
        subplt.legend()
    plt.show()

if __name__ == '__main__':
    xml_path = curr_dir + "\config\kpi.xml"
    files_lists = xml_parser.read_xml(xml_path)

    for files_list in files_lists:
        all_datas = []
        #print read_kpi_data.read_datas(files_list)
        plot_name,plot_datas = read_kpi_data.read_datas(files_list)
        datas = plot_datas.values()
        all_datas.extend(datas)

        for d in all_datas:
            multiprocessing.Process(target=graph_display,args=(plot_name,d)).start()

数据可视化效果：