Sqlite3 - python - ftrace 数值统计与分析

引用

• 菜鸟教程 - SQLite
• Python3 操作SQLite数据库
• python开发_sqlite3_绝对完整
• 菜鸟教程-Python JSON
• sqlite3 DB to excel file

---

一. sample code

{
  "col_info": ["ID", "FUNCTION_NAME", "ABS_TIME", "LATENCY", "FLAG", "VAR1_NAME", "VAR1_VAL"],
  "table_info": ["detail_info", "summary_info"]
}

import sqlite3
import os
import json
import time


class INFO:
    name = None
    abs_time = 0
    suspend_time = 0
    flag = None


class STACK():
    def __init__(self):
        self.items = []

    def is_empty(self):
        return self.items == []

    def push(self, item):
        self.items.append(item)

    def pop(self):
        return self.items.pop()

    def peek(self):
        return self.items[len(self.items) - 1]

    def size(self):
        return len(self.items)


def parse_line(str):
    t_str = str.split("|")
    if len(t_str) != 3:
        return 1, 1

    info = INFO()

    # get abs time
    info.abs_time = t_str[0]

    # get suspend time
    if "us" in t_str[1]:
        t_str2 = t_str[1].split()
        info.suspend_time = t_str2[-2]

    # get function name
    if ";" in t_str[2]:
        info.name = t_str[2][:-4].replace(' ', '')
    elif "{" in t_str[2]:
        info.name = t_str[2][:-5].replace(' ', '')
        info.flag = "{"
    elif "}" in t_str[2]:
        info.flag = "}"

    # print("{}, {}, {}, {}".format(info.name, info.abs_time, info.suspend_time, info.flag))
    return 0, info


def get_funciton_latency_info(sqlite3_cur, table_name, fucntion_name_col, latency_col, function):
    sql = "SELECT COUNT({}) FROM {} WHERE {} LIKE '{}';".format(latency_col, table_name, fucntion_name_col, function)
    sqlite3_cur.execute(sql)
    count = sqlite3_cur.fetchone()[0]

    sql = "SELECT MAX({}) FROM {} WHERE {} LIKE '{}';".format(latency_col, table_name, fucntion_name_col, function)
    sqlite3_cur.execute(sql)
    max = sqlite3_cur.fetchone()[0]

    sql = "SELECT MIN({}) FROM {} WHERE {} LIKE '{}';".format(latency_col, table_name, fucntion_name_col, function)
    sqlite3_cur.execute(sql)
    min = sqlite3_cur.fetchone()[0]

    sql = "SELECT AVG({}) FROM {} WHERE {} LIKE '{}';".format(latency_col, table_name, fucntion_name_col, function)
    sqlite3_cur.execute(sql)
    avg = sqlite3_cur.fetchone()[0]

    sql = "SELECT SUM({}) FROM {} WHERE {} LIKE '{}';".format(latency_col, table_name, fucntion_name_col, function)
    sqlite3_cur.execute(sql)
    sum = sqlite3_cur.fetchone()[0]

    print("{}() - count:{}, max:{}, min:{}, avg:{:.6f}, sum:{:.6f}".format(function, count, max, min, avg, sum))


def update_the_function_latency(sqlite3_conn, table_name, update_col):
    flush_time = 0
    stack = STACK()
    # query the table data
    for row in sqlite3_conn.execute("SELECT * FROM {}".format(table_name)):
        if row[4] == "{":
            row_id = row[0]
            stack.push(row_id)
            continue

        if stack.is_empty() is not True:
            if row[4] == "}":
                latency = row[3]
                row_id = stack.pop()
                sql = "UPDATE {} SET {} = {} where ID={}".format(table_name, update_col, latency, row_id)
                sqlite3_conn.execute(sql)

                flush_time = flush_time + 1
                if flush_time % 1000 == 0:
                    sqlite3_conn.commit()

    sqlite3_conn.commit()


def get_latency_between_two_function(sqlite3_conn, table_name, func1, func2):
    latency_list = []
    abs_time1 = 0
    # query the table data
    for row in sqlite3_conn.execute("SELECT * FROM {}".format(table_name)):
        if row[1] == func1:
            abs_time1 = row[2]
            continue

        if abs_time1 != 0:
            if row[1] == func2:
                abs_time2 = row[2]
                latency = abs_time2 - abs_time1
                abs_time1 = 0

                latency_list.append(round(latency, 6))
                # latency_list.append(float("{:.6f}".format(latency)))

    print(latency_list)
    return latency_list


def creat_sqlite3_db(sqlite_db, table_name, col_format):
    # delete the old DB file
    if os.path.exists(sqlite_db) is True:
        os.remove(sqlite_db)

    # create a sqlite
    conn = sqlite3.connect(sqlite_db)
    print("Opened database successfully")

    # create a cursor
    cur = conn.cursor()

    # execute a SQL CMD to create a table
    sql = " CREATE TABLE IF NOT EXISTS {} ({} INTEGER PRIMARY KEY NOT NULL," \
          "{} TEXT,{} REAL,{} REAL,{} CHAR(1),{} TEXT,{} INT);".format(table_name,
                                                                       col_format[0], col_format[1], col_format[2],
                                                                       col_format[3], col_format[4], col_format[5],
                                                                       col_format[6])
    cur.execute(sql)
    print("Table created successfully")
    conn.commit()

    return conn, cur


def parse_ftrace_log(sqlite3_conn, sqlite3_cur, table_name, col_format, ftrace):
    flush_time = 0
    # parse the ftrace file to sqlite3
    with open(ftrace, 'r', encoding='UTF-8') as f:
        line = f.readline()
        while line is not None and line != '':
            ret, info = parse_line(line)
            if ret == 0:
                # insert some test data
                sql = "INSERT INTO {} ({}, {}, {}, {}) VALUES (?, ?, ?, ?)".format(
                    table_name, col_format[1], col_format[2], col_format[3], col_format[4]
                )
                sqlite3_cur.execute(sql, (info.name, info.abs_time, info.suspend_time, info.flag))
            line = f.readline()

        if flush_time % 1000 == 0:
            sqlite3_conn.commit()

    sqlite3_conn.commit()
    print('Records created successfully')


def get_function_up_val(sqlite3_conn, table_name, val):
    func_list = set()
    # query the table data
    for row in sqlite3_conn.execute("SELECT * FROM {}".format(table_name)):
        if row[3] >= val and row[1] != None:
            func_list.add(row[1])

    print(func_list)


def get_info_between_two_function(sqlite3_conn, table_name, col_info, func1, func2, event1):
    func_time = 0
    start_flag = False
    func_times_list = []

    # query the table data
    sql = "SELECT * FROM {}".format(table_name)
    for row in sqlite3_conn.execute(sql):
        if row[col_info.index("FUNCTION_NAME")] == func1 and start_flag is False:
            start_flag = True
            continue

        if start_flag is True:
            if row[col_info.index("FUNCTION_NAME")] == event1:
                func_time = func_time + 1

            if row[col_info.index("FUNCTION_NAME")] == func2:
                start_flag = False
                func_times_list.append(func_time)
                func_time = 0

    print(func_times_list)


def add_new_col_in_table(sqlite3_conn, sqlite3_cur, table_name, col_name, col_type):
    sql = " CREATE TABLE IF NOT EXISTS {} ({} INTEGER PRIMARY KEY NOT NULL);".format(table_name, "ID")
    sqlite3_cur.execute(sql)
    print("Table {}: created successfully".format(table_name))

    sql = "ALTER TABLE {} ADD COLUMN '{}' {}".format(table_name, col_name, col_type)
    sqlite3_cur.execute(sql)
    print("col {}: alter new col successfully".format(col_name))

    sqlite3_conn.commit()


def add_list_to_col(sqlite3_conn, sqlite3_cur, table_name, col_name, val_list):
    flush = 0
    for val in val_list:
        # insert some test data
        sql = "INSERT INTO {} ({}) VALUES (?)".format(
            table_name, col_name)
        sqlite3_cur.execute(sql, (val,))
        flush = flush + 1
        if flush % 1000 == 0:
            sqlite3_conn.commit()

    sqlite3_conn.commit()


def main():
    db_name = "test2.db"

    # parse the json file
    with open("db_configture.json", 'r') as load_f:
        json_data = json.load(load_f)
        print(json_data)

    col_format = json_data["col_info"]
    table_name = json_data["table_info"][0]

    # create a sqlite3 database
    conn, cur = creat_sqlite3_db(db_name, table_name, col_format)
    last_time = time.time()
    print(last_time)

    # parse the ftrace log file to sqlite3
    ftrace_log_name = "test.log"
    parse_ftrace_log(conn, cur, table_name, col_format, ftrace_log_name)

    # update the function latency with "{", "}"
    update_the_function_latency(conn, table_name, col_format[3])

    # get the latency between two function according abs_time
    latency_list = get_latency_between_two_function(conn, table_name, "tty_insert_flip_string_fixed_flag",
                                                    "tty_ldisc_receive_buf")
    add_new_col_in_table(conn, cur, "test", "t1", "REAL")
    add_list_to_col(conn, cur, "test", "t1", latency_list)

    # get function latency information, such as, max, min, avg, sum ...
    get_funciton_latency_info(cur, table_name, col_format[1], col_format[3], "tty_ldisc_ref")

    # get the information up specific value
    get_function_up_val(conn, table_name, 25)

    # get event information between two function
    get_info_between_two_function(conn, table_name, col_format,
                                  "tty_insert_flip_string_fixed_flag", "tty_insert_flip_string_fixed_flag",
                                  "tty_ldisc_ref")

    # creat new col in a table
    add_new_col_in_table(conn, cur, table_name, "t1", "INT")
    add_new_col_in_table(conn, cur, "test1", "t1", "INT")
    add_new_col_in_table(conn, cur, "test1", "t2", "TEXT")
    add_new_col_in_table(conn, cur, "test2", "t2", "TEXT")

    conn.close()


if __name__ == '__main__':
    main()

import DaPy as dp

sheet = dp.read('test2.db')
sheet.save('db2excel.xls')

二. ftrace

# tracer: function_graph
#
#     TIME        CPU  DURATION                  FUNCTION CALLS
#      |          |     |   |                     |   |   |   |
  528.787555 |     1)   1.687 us    |  tty_ldisc_ref();
  528.787557 |     1)   0.146 us    |  tty_ldisc_ref();
  528.787576 |     1)   0.436 us    |  tty_insert_flip_string_fixed_flag();
  528.787643 |     0)               |  flush_to_ldisc() {
  528.787644 |     0)               |    tty_port_default_receive_buf() {
  528.787644 |     0)   0.250 us    |      tty_ldisc_ref();
  528.787644 |     0)   4.625 us    |      tty_ldisc_receive_buf();
  528.787649 |     0)   5.375 us    |    }
  528.787649 |     0)   6.883 us    |  }

三. 数值统计与分析结果

 

四. sqlite3格式