一、看一下sysbench自带的lua脚本
commonn.lua脚本。
都是一些，创建表和插入数据的准备语句。
比较重要的是：一些变量的默认值在这里定义了。
function create_insert(table_id)
local index_name
local i
local j
local query
if (oltp_secondary) then
index_name = "KEY xid"
else
index_name = "PRIMARY KEY"
end
if (pgsql_variant == 'redshift') then
auto_inc_type = "INTEGER IDENTITY(1,1)"
else
auto_inc_type = "SERIAL"
end
i = table_id
print("Creating table 'sbtest" .. i .. "'...")
if ((db_driver == "mysql") or (db_driver == "attachsql")) then
query = [[
CREATE TABLE sbtest]] .. i .. [[ (
id INTEGER UNSIGNED NOT NULL ]] ..
((oltp_auto_inc and "AUTO_INCREMENT") or "") .. [[,
k INTEGER UNSIGNED DEFAULT '0' NOT NULL,
c CHAR(120) DEFAULT '' NOT NULL,
pad CHAR(60) DEFAULT '' NOT NULL,
]] .. index_name .. [[ (id)
) /*! ENGINE = ]] .. mysql_table_engine ..
" MAX_ROWS = " .. myisam_max_rows .. " */ " ..
(mysql_table_options or "")
elseif (db_driver == "pgsql") then
query = [[
CREATE TABLE sbtest]] .. i .. [[ (
id ]] .. auto_inc_type .. [[ NOT NULL,
k INTEGER DEFAULT '0' NOT NULL,
c CHAR(120) DEFAULT '' NOT NULL,
pad CHAR(60) DEFAULT '' NOT NULL,
]] .. index_name .. [[ (id)
) ]]
elseif (db_driver == "drizzle") then
query = [[
CREATE TABLE sbtest (
id INTEGER NOT NULL ]] .. ((oltp_auto_inc and "AUTO_INCREMENT") or "") .. [[,
k INTEGER DEFAULT '0' NOT NULL,
c CHAR(120) DEFAULT '' NOT NULL,
pad CHAR(60) DEFAULT '' NOT NULL,
]] .. index_name .. [[ (id)
) ]]
else
print("Unknown database driver: " .. db_driver)
return 1
end
db_query(query)
print("Inserting " .. oltp_table_size .. " records into 'sbtest" .. i .. "'")
if (oltp_auto_inc) then
db_bulk_insert_init("INSERT INTO sbtest" .. i .. "(k, c, pad) VALUES")
else
db_bulk_insert_init("INSERT INTO sbtest" .. i .. "(id, k, c, pad) VALUES")
end
local c_val
local pad_val
for j = 1,oltp_table_size do
c_val = sb_rand_str([[
###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]])
pad_val = sb_rand_str([[
###########-###########-###########-###########-###########]])
if (oltp_auto_inc) then
db_bulk_insert_next("(" .. sb_rand(1, oltp_table_size) .. ", '".. c_val .."', '" .. pad_val .. "')")
else
db_bulk_insert_next("("..j.."," .. sb_rand(1, oltp_table_size) .. ",'".. c_val .."', '" .. pad_val .. "' )")
end
end
db_bulk_insert_done()
if oltp_create_secondary then
print("Creating secondary indexes on 'sbtest" .. i .. "'...")
db_query("CREATE INDEX k_" .. i .. " on sbtest" .. i .. "(k)")
end
end
function prepare()
local query
local i
local j
set_vars()
db_connect()
for i = 1,oltp_tables_count do
create_insert(i)
end
return 0
end
function cleanup()
local i
set_vars()
for i = 1,oltp_tables_count do
print("Dropping table 'sbtest" .. i .. "'...")
db_query("DROP TABLE IF EXISTS sbtest".. i )
end
end
// 这里是设置变量，可以看到一些默认变量，我们不指定的时候，使用的就是这些初始值
function set_vars()
oltp_table_size = tonumber(oltp_table_size) or 10000 // 默认表size
oltp_range_size = tonumber(oltp_range_size) or 100 // 范围
oltp_tables_count = tonumber(oltp_tables_count) or 1 // 表数量
oltp_point_selects = tonumber(oltp_point_selects) or 10 // 查询次数
oltp_simple_ranges = tonumber(oltp_simple_ranges) or 1
oltp_sum_ranges = tonumber(oltp_sum_ranges) or 1
oltp_order_ranges = tonumber(oltp_order_ranges) or 1
oltp_distinct_ranges = tonumber(oltp_distinct_ranges) or 1
oltp_index_updates = tonumber(oltp_index_updates) or 1
oltp_non_index_updates = tonumber(oltp_non_index_updates) or 1
oltp_delete_inserts = tonumber(oltp_delete_inserts) or 1
if (oltp_range_selects == 'off') then
oltp_range_selects = false
else
oltp_range_selects = true
end
if (oltp_auto_inc == 'off') then
oltp_auto_inc = false
else
oltp_auto_inc = true
end
if (oltp_read_only == 'on') then
oltp_read_only = true
else
oltp_read_only = false
end
if (oltp_write_only == 'on') then
oltp_write_only = true
else
oltp_write_only = false
end
if (oltp_read_only and oltp_write_only) then
error("--oltp-read-only and --oltp-write-only are mutually exclusive")
end
if (oltp_skip_trx == 'on') then
oltp_skip_trx = true
else
oltp_skip_trx = false
end
if (oltp_create_secondary == 'off') then
oltp_create_secondary = false
else
oltp_create_secondary = true
end
if (pgsql_variant == 'redshift') then
oltp_create_secondary = false
oltp_delete_inserts = 0
end
这是我们使用的oltp.lua
可以看到，如果指定了只读模式，则只会执行各种只读语句，而且还可以控制各种读语句，以及他们的次数。
同理，如果指定了只写模式，则也指定了update的次数，以及索引非索引更新，还有delete。
pathtest = string.match(test, "(.*/)")
// 引入common.lua
if pathtest then
dofile(pathtest .. "common.lua")
else
require("common")
end
function thread_init()
set_vars()
// 如果是myisam则，定义锁住所有测试表的语句
if (((db_driver == "mysql") or (db_driver == "attachsql")) and mysql_table_engine == "myisam") then
local i
local tables = {}
for i=1, oltp_tables_count do
tables[i] = string.format("sbtest%i WRITE", i)
end
begin_query = "LOCK TABLES " .. table.concat(tables, " ,")
commit_query = "UNLOCK TABLES"
else
// 否则，开启事务语句
begin_query = "BEGIN"
commit_query = "COMMIT"
end
end
function get_range_str()
local start = sb_rand(1, oltp_table_size)
return string.format(" WHERE id BETWEEN %u AND %u",
start, start + oltp_range_size - 1)
end
function event()
local rs
local i
local table_name
local c_val
local pad_val
local query
table_name = "sbtest".. sb_rand_uniform(1, oltp_tables_count)
// 是否跳过事务
if not oltp_skip_trx then
db_query(begin_query)
end
// 非只写模式
if not oltp_write_only then
for i=1, oltp_point_selects do
rs = db_query("SELECT c FROM ".. table_name .." WHERE id=" ..
sb_rand(1, oltp_table_size))
end
if oltp_range_selects then
for i=1, oltp_simple_ranges do
rs = db_query("SELECT c FROM ".. table_name .. get_range_str())
end
for i=1, oltp_sum_ranges do
rs = db_query("SELECT SUM(K) FROM ".. table_name .. get_range_str())
end
for i=1, oltp_order_ranges do
rs = db_query("SELECT c FROM ".. table_name .. get_range_str() ..
" ORDER BY c")
end
for i=1, oltp_distinct_ranges do
rs = db_query("SELECT DISTINCT c FROM ".. table_name .. get_range_str() ..
" ORDER BY c")
end
end
end
// // 非只读模式
if not oltp_read_only then
for i=1, oltp_index_updates do
rs = db_query("UPDATE " .. table_name .. " SET k=k+1 WHERE id=" .. sb_rand(1, oltp_table_size))
end
for i=1, oltp_non_index_updates do
c_val = sb_rand_str("###########-###########-###########-###########-###########-###########-###########-###########-###########-###########")
query = "UPDATE " .. table_name .. " SET c='" .. c_val .. "' WHERE id=" .. sb_rand(1, oltp_table_size)
rs = db_query(query)
if rs then
print(query)
end
end
for i=1, oltp_delete_inserts do
i = sb_rand(1, oltp_table_size)
rs = db_query("DELETE FROM " .. table_name .. " WHERE id=" .. i)
c_val = sb_rand_str([[
###########-###########-###########-###########-###########-###########-###########-###########-###########-###########]])
pad_val = sb_rand_str([[
###########-###########-###########-###########-###########]])
rs = db_query("INSERT INTO " .. table_name .. " (id, k, c, pad) VALUES " .. string.format("(%d, %d, '%s', '%s')",i, sb_rand(1, oltp_table_size) , c_val, pad_val))
end
end -- oltp_read_only
// 跳过事务
if not oltp_skip_trx then
db_query(commit_query)
end
end
总体而言：oltp.lua脚本是一个综合脚本，可以指定一个事务里的执行各种语句细节。
可以看到，除了oltp.lua脚本，还有其他脚本，比如，纯select脚本，insert脚本等等，这些脚本相对oltp.lua这种综合脚本就比较简单，功能比较单一。
[root@localhost sysbench]# ls ./tests/include/oltp_legacy/
bulk_insert.lua delete.lua mytest.lua oltp_simple.lua select.lua select_random_ranges.lua update_non_index.lua
common.lua insert.lua oltp.lua parallel_prepare.lua select_random_points.lua update_index.lua
二、自定义脚本
在一些其他情况，我们不想用它的脚本，要高度灵活贴近业务。
比如，我们的核心业务，一个事务里会执行:
start transaction;
select * from sbtest1 where id = '';
update sbtest1 set k = k where id = id;
insert into sbtest1 (k, c, pad) values (3, 'world', 'worlwd')
commit;
然后我们需要测试，这个业务的极限qps到底是多少
使用自定义lua脚本。
pathtest = string.match(test, "(.*/)")
// 必须引入common
if pathtest then
dofile(pathtest .. "common.lua")
else
require("common")
end
function thread_init()
set_vars()
// 这里可以修改，不过也无所谓
if (((db_driver == "mysql") or (db_driver == "attachsql")) and mysql_table_engine == "myisam") then
local i
local tables = {}
for i=1, oltp_tables_count do
tables[i] = string.format("sbtest%i WRITE", i)
end
begin_query = "LOCK TABLES " .. table.concat(tables, " ,")
commit_query = "UNLOCK TABLES"
else
begin_query = "BEGIN"
commit_query = "COMMIT"
end
end
function event()
local vid
local vid1
// 生成随机值
vid = sb_rand_uniform(1,1000)
vid1 = sb_rand_uniform(500,10000)
// 开启事务
db_query(begin_query)
rs = db_query("SELECT * FROM sbtest1 WHERE id=" .. vid1)
db_query("update sbtest1 set k = " ..vid1 .." where id = " .. vid)
db_query("insert into sbtest1 (k, c, pad) values (3, 'world', 'worlwd')")
// 结束事务
db_query(commit_query)
end
执行测试。
测试结果：并发数100，表数据量1000。
qps：4千多。tps：934。错误数：0个每秒。
平均每个请求延迟106ms。
[root@localhost sysbench]# sysbench ./tests/include/oltp_legacy/mytest.lua --mysql-host=192.168.220.1 --mysql-port=3306 --mysql-user=root --mysql-password=123456 --oltp-test-mode=complex --threads=100 --time=30 --report-interval=10 run
sysbench 1.0.17 (using bundled LuaJIT 2.1.0-beta2)
Running the test with following options:
Number of threads: 100
Report intermediate results every 10 second(s)
Initializing random number generator from current time
Initializing worker threads...
Threads started!
[ 10s ] thds: 100 tps: 836.81 qps: 4196.36 (r/w/o: 838.11/936.69/2421.56) lat (ms,95%): 331.91 err/s: 0.00 reconn/s: 0.00
[ 20s ] thds: 100 tps: 1041.47 qps: 5234.76 (r/w/o: 1050.17/1051.27/3133.32) lat (ms,95%): 257.95 err/s: 0.00 reconn/s: 0.00
[ 30s ] thds: 100 tps: 932.41 qps: 4657.13 (r/w/o: 932.01/928.51/2796.62) lat (ms,95%): 314.45 err/s: 0.00 reconn/s: 0.00
SQL statistics:
queries performed:
read: 28209
write: 29208
other: 83628
total: 141045
transactions: 28209 (934.54 per sec.)
queries: 141045 (4672.70 per sec.)
ignored errors: 0 (0.00 per sec.)
reconnects: 0 (0.00 per sec.)
General statistics:
total time: 30.1834s
total number of events: 28209
Latency (ms):
min: 35.93
avg: 106.65
max: 1069.06
95th percentile: 297.92
sum: 3008469.21
Threads fairness:
events (avg/stddev): 282.0900/4.54
execution time (avg/stddev): 30.0847/0.07
我们如果，把并发增大，加大连接数。
把连接数设置成1000个，默认200个。
set global max_connections=1000
执行测试。
测试结果：并发数800，表数据量1000。
qps：9千。tps：1786。错误数：0个每秒。
平均每个请求延迟442ms。
可以看到：增加连接数，可以把qps的瓶颈压出来，但是延迟时间会增加。
[root@localhost sysbench]# sysbench ./tests/include/oltp_legacy/mytest.lua --mysql-host=192.168.220.1 --mysql-port=3306 --mysql-user=root --mysql-password=123456 --oltp-test-mode=complex --threads=800 --time=30 --report-interval=10 run
sysbench 1.0.17 (using bundled LuaJIT 2.1.0-beta2)
Running the test with following options:
Number of threads: 800
Report intermediate results every 10 second(s)
Initializing random number generator from current time
Initializing worker threads...
Threads started!
[ 10s ] thds: 800 tps: 1934.75 qps: 9841.96 (r/w/o: 1989.12/1944.24/5908.60) lat (ms,95%): 877.61 err/s: 0.00 reconn/s: 0.00
[ 20s ] thds: 800 tps: 1832.52 qps: 9159.91 (r/w/o: 1840.42/1825.72/5493.77) lat (ms,95%): 893.56 err/s: 0.00 reconn/s: 0.00
[ 30s ] thds: 800 tps: 1628.91 qps: 8224.11 (r/w/o: 1645.20/1652.80/4926.11) lat (ms,95%): 960.30 err/s: 0.00 reconn/s: 0.00
SQL statistics:
queries performed:
read: 54769
write: 54769
other: 164307
total: 273845
transactions: 54769 (1786.52 per sec.)
queries: 273845 (8932.62 per sec.)
ignored errors: 0 (0.00 per sec.)
reconnects: 0 (0.00 per sec.)
General statistics:
total time: 30.6553s
total number of events: 54769
Latency (ms):
min: 57.82
avg: 442.69
max: 2044.83
95th percentile: 943.16
sum: 24245431.84
Threads fairness:
events (avg/stddev): 68.4613/5.33
execution time (avg/stddev): 30.3068/0.23
三、更准确的是在数据库层面统计，qps。
在数据库统计，可以在生产中都统计的到数据。
写一个脚本：定时统计，已发送queue数，thread连接数，正在跑的连接数。
#!/bin/bash
while true
do
mysqladmin -uroot -p123456 -h 192.168.220.1 ext | awk '/Queries/{q=$4}/Threads_connected/{c=$4}/Threads_running/{r=$4}END{printf("%d %d %d\n",q,c,r)}' >> status.txt
sleep 1
done
然后，跑前面的测试脚本。
可以看到，线程数是800多，跟我们sysbench指定的线程数一致。
[root@localhost sysbench]# more status.txt
9429389 804 801
9433265 804 801
9445655 804 791
9458973 804 798
9469785 804 630
9484325 804 553
9495997 804 799
9506675 804 800
9519388 804 801
9528600 804 801
9535616 804 801
9544293 804 801
9554229 804 801
9559486 804 801
9567507 804 801
9577735 804 801
9586993 804 801
9594322 804 801
9601483 804 801
9608285 804 801
9611927 804 801
9620066 804 801
9630438 804 801
9640654 804 801
9647684 804 801
9653961 804 801
9663624 804 449
9674744 804 550
然后，把queue数，下一行，减上一行，统计每秒qps。
可以看到，大概的pqs是8000到1万。跟我们的sysbench统计的差不多。
awk '{q=$1-last;last=$1}{printf(" %d %d %d\n", q,$2,$3)}' status.txt
3876 804 801
12390 804 791
13318 804 798
10812 804 630
14540 804 553
11672 804 799
10678 804 800
12713 804 801
9212 804 801
7016 804 801
8677 804 801
9936 804 801
5257 804 801
8021 804 801
10228 804 801
9258 804 801
7329 804 801
7161 804 801
6802 804 801
3642 804 801
8139 804 801
10372 804 801
10216 804 801
7030 804 801
6277 804 801
9663 804 449
然后再生成可视化图表。
image.png