Greenplum性能测试
本文主要记录Greenplum的性能测试过程，仅供参考
测试环境
Greenplum版本
5.10.1
部署架构图
节点部署说明
测试机器使用vmware虚拟机+Centos7.4
| IP | 节点类型 | 说明 |
|---|---|---|
| 192.168.26.21 | Master | 1个Master节点 |
| 192.168.26.22 | Segment | 1个Primary节点 + 1个Mirror节点 |
| 192.168.26.23 | Segment | 1个Primary节点 + 1个Mirror节点 |
| 192.168.26.24 | Segment+ Master(standby) | 1个Primary节点 + 1个Mirror节点 + 1个Master备用节点 |
硬件配置说明
| 硬件 | 值 |
|---|---|
| 处理器 | 4核 |
| 内存 | 2G |
| 磁盘 | 30G SSD |
测试用例
用例一：测试单表单线程写入数据性能
建表语句
create table t_person4insert
(
id bigint,
name varchar(255)
)
distributed by(id);使用Greenplum官方JDBC驱动写入数据，总共写入100w条数据，每1000条提交一次
class PersonThread implements Runnable {
@Override
public void run() {
long begin = System.currentTimeMillis();
try {
Class.forName("com.pivotal.jdbc.GreenplumDriver");
try (Connection conn = DriverManager.getConnection("jdbc:pivotal:greenplum://192.168.26.21:5432;DatabaseName=testDB", "gpadmin", "gpadmin");) {
String sql = "insert into t_person (id,name) values (?,?)";
try(PreparedStatement ps = conn.prepareStatement(sql);){
conn.setAutoCommit(false); // 把自动提交
System.out.println("Opened database successfully");
long id = 1;
String namePrefix = "atlasdata";
for(int i=0; i<1000; i++) {
for(int j=0; j<1000; j++) {
ps.setLong(1, id);
ps.setString(2, namePrefix);
ps.addBatch();
id++;
}
ps.executeBatch();
conn.commit();
// System.out.println("person insert count = " + (id - 1));
}
} catch (Exception e) {
conn.rollback();
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
}
} catch (Exception e) {
e.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("person cost = " + (end - begin));
}
}测试结果
person4insert cost = 1945021testDB=# select pg_size_pretty(pg_relation_size('t_person4insert'));
pg_size_pretty
----------------
50 MB
(1 row)计算吞吐量 = 50 / 1945021 * 1000 = 0.0257 M/s
用例二：测试多表多线程写入数据性能
建表语句
create table t_person4insert1
(
id bigint,
name varchar(255)
)
distributed by(id);
create table t_person4insert2
(
id bigint,
name varchar(255)
)
distributed by(id);
create table t_person4insert3
(
id bigint,
name varchar(255)
)
distributed by(id);
create table t_person4insert4
(
id bigint,
name varchar(255)
)
distributed by(id);使用Greenplum官方JDBC驱动写入数据，启动4个线程分别写入t_person4insert1、t_person4insert2、t_person4insert3、t_person4insert4、总共写入400w条数据，每个线程写入100w条数据，每1000条提交一次
public static void main(String[] args) throws Exception {
long begin = System.currentTimeMillis();
Thread t1 = new Thread(new PersonXThread("1"));
Thread t2 = new Thread(new PersonXThread("2"));
Thread t3 = new Thread(new PersonXThread("3"));
Thread t4 = new Thread(new PersonXThread("4"));
t1.start();
t2.start();
t3.start();
t4.start();
t1.join();
t2.join();
t3.join();
t4.join();
long end = System.currentTimeMillis();
System.out.println("total cost = " + (end - begin));
}
}
class PersonXThread implements Runnable {
private String index;
public PersonXThread(String index) {
this.index = index;
}
@Override
public void run() {
long begin = System.currentTimeMillis();
try {
Class.forName("com.pivotal.jdbc.GreenplumDriver");
try (Connection conn = DriverManager.getConnection("jdbc:pivotal:greenplum://192.168.26.21:5432;DatabaseName=testDB", "gpadmin", "gpadmin");) {
String sql = "insert into t_person4insert" + index + " (id,name) values (?,?)";
try(PreparedStatement ps = conn.prepareStatement(sql);){
conn.setAutoCommit(false); // 把自动提交
System.out.println("Opened database successfully");
long id = 1;
String namePrefix = "atlasdata";
for(int i=0; i<1000; i++) {
for(int j=0; j<1000; j++) {
ps.setLong(1, id);
ps.setString(2, namePrefix);
ps.addBatch();
id++;
}
ps.executeBatch();
conn.commit();
// System.out.println("person4insert " + index + " insert count = " + (id - 1));
}
} catch (Exception e) {
conn.rollback();
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
}
} catch (Exception e) {
e.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("person4insert " + index + " cost = " + (end - begin));
}
}测试结果
person4insert 2 cost = 1608538
person4insert 1 cost = 1629877
person4insert 3 cost = 1644159
person4insert 4 cost = 1698275
total cost = 1698277testDB=# select pg_size_pretty(pg_relation_size('t_person4insert1'));
pg_size_pretty
----------------
50 MB
(1 row)
testDB=# select pg_size_pretty(pg_relation_size('t_person4insert2'));
pg_size_pretty
----------------
50 MB
(1 row)
testDB=# select pg_size_pretty(pg_relation_size('t_person4insert3'));
pg_size_pretty
----------------
50 MB
(1 row)
testDB=# select pg_size_pretty(pg_relation_size('t_person4insert4'));
pg_size_pretty
----------------
50 MB
(1 row)
计算吞吐量 = 50 * 4 / 1698277 * 1000 = 0.117 M/s
用例三：测试单表多线程写入数据性能
建表语句
create table t_person4insertrange
(
id bigint,
name varchar(255)
)
distributed by(id);使用Greenplum官方JDBC驱动写入数据，启动4个线程分别写入t_person4insertrange、总共写入400w条数据，每个线程写入100w条数据，每1000条提交一次
public static void main(String[] args) throws Exception {
long begin = System.currentTimeMillis();
Thread t1 = new Thread(new PersonRangegThread(1));
Thread t2 = new Thread(new PersonRangegThread(1000001));
Thread t3 = new Thread(new PersonRangegThread(2000001));
Thread t4 = new Thread(new PersonRangegThread(3000001));
t1.start();
t2.start();
t3.start();
t4.start();
t1.join();
t2.join();
t3.join();
t4.join();
long end = System.currentTimeMillis();
System.out.println("total cost = " + (end - begin));
}
}
class PersonRangegThread implements Runnable {
private long startPos;
public PersonRangegThread(long begin) {
this.startPos = begin;
}
@Override
public void run() {
long begin = System.currentTimeMillis();
try {
Class.forName("com.pivotal.jdbc.GreenplumDriver");
try (Connection conn = DriverManager.getConnection("jdbc:pivotal:greenplum://192.168.26.21:5432;DatabaseName=testDB", "gpadmin", "gpadmin");) {
String sql = "insert into t_person4insertrange (id,name) values (?,?)";
try(PreparedStatement ps = conn.prepareStatement(sql);){
conn.setAutoCommit(false); // 把自动提交
System.out.println("Opened database successfully");
long id = this.startPos;
String namePrefix = "atlasata";
for(int i=0; i<1000; i++) {
for(int j=0; j<1000; j++) {
ps.setLong(1, id);
ps.setString(2, namePrefix);
ps.addBatch();
id++;
}
ps.executeBatch();
conn.commit();
//System.out.println("t_person4insertrange startPos = " + startPos + " insert count = " + (id - startPos));/
}
} catch (Exception e) {
conn.rollback();
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
}
} catch (Exception e) {
e.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("t_person4insertrange startPos = " + startPos + " cost = " + (end - begin));
}
}测试结果
t_person4insertrange startPos = 1 cost = 1615610
t_person4insertrange startPos = 3000001 cost = 1632609
t_person4insertrange startPos = 2000001 cost = 1655256
t_person4insertrange startPos = 1000001 cost = 1660568
total cost = 1660570
testDB=# select pg_size_pretty(pg_relation_size('t_person4insertrange '));
pg_size_pretty
----------------
199 MB
(1 row)计算吞吐量 = 199 / 1660570* 1000 = 0.112 M/s
用例四：测试条件查询性能
建表语句
create table t_person
(
id bigint,
name varchar(255)
)
distributed by(id);
create table t_person1
(
id bigint primary key,
name varchar(255)
)
distributed by(id);使用Greenplum官方JDBC驱动往t_person、t_person1写入数据，总共写入100w条数据，每1000条提交一次
无索引查询
testDB=# select * from t_person where id = 965813;
id | name
--------+-----------------
965813 | atlasdata965813
(1 row)
Time: 43.089 ms索引查询
testDB=# select * from t_person1 where id = 965813;
id | name
--------+-----------------
965813 | atlasdata965813
(1 row)
Time: 24.280 ms测试用例五：测试关联查询性能
建表语句
create table t_person
(
id bigint,
name varchar(255)
)
distributed by(id);
create table t_person_address2
(
id bigint,
address varchar(255),
person_id bigint
)
distributed by(id);
create table t_person_address3
(
id bigint,
address varchar(255),
person_id bigint
)
distributed by(person_id);
create table t_person_address4
(
id bigint,
address varchar(255),
person_id bigint
)
distributed by(person_id);
CREATE INDEX idex_pd4_pid ON t_person_address4 (person_id); 使用Greenplum官方JDBC驱动往t_person、t_person_address2、t_person_address3、t_person_address4写入数据，总共写入100w条数据，每1000条提交一次
class PersonAddressThread implements Runnable {
@Override
public void run() {
long begin = System.currentTimeMillis();
try {
Class.forName("com.pivotal.jdbc.GreenplumDriver");
try (Connection conn = DriverManager.getConnection("jdbc:pivotal:greenplum://192.168.26.21:5432;DatabaseName=testDB", "gpadmin", "gpadmin");) {
String sql = "insert into t_person_address (id,address,person_id) values (?,?,?)";
try(PreparedStatement ps = conn.prepareStatement(sql);){
conn.setAutoCommit(false); // 把自动提交
System.out.println("Opened database successfully");
long id = 1;
String addressPrefix = "guangzhou@";
for(int i=0; i<1000; i++) {
for(int j=0; j<1000; j++) {
ps.setLong(1, id);
ps.setString(2, addressPrefix + id);
ps.setLong(3, id+1);
ps.addBatch();
id++;
}
ps.executeBatch();
conn.commit();
// System.out.println("person address insert count = " + (id - 1));
}
} catch (Exception e) {
conn.rollback();
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
}
} catch (Exception e) {
e.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("person address cost = " + (end - begin));
}
}关联查询 分片键==关联键 b表关联键不带索引
testDB=# select * from t_person a join t_person_address3 b on a.id = b.person_id;
省略查询结果
Time: 1519.862 ms关联查询 分片键==关联键 b表关联键带索引
testDB=# select * from t_person a join t_person_address4 b on a.id = b.person_id;
省略查询结果
Time: 1737.071 ms关联查询 分片键!=关联键
testDB=# select * from t_person a join t_person_address2 b on a.id = b.person_id;
省略查询结果
Time: 1853.398 ms查看查询计划
testDB=# explain select * from t_person a join t_person_address3 b on a.id = b.person_id;
QUERY PLAN
----------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.00 rows=1 width=46)
-> Hash Join (cost=0.00..862.00 rows=1 width=46)
Hash Cond: t_person_address3.person_id = t_person.id
-> Table Scan on t_person_address3 (cost=0.00..431.00 rows=1 width=27)
-> Hash (cost=431.00..431.00 rows=1 width=19)
-> Table Scan on t_person (cost=0.00..431.00 rows=1 width=19)
Optimizer status: PQO version 2.65.0
(7 rows)
Time: 10.651 ms
testDB=# explain select * from t_person a join t_person_address4 b on a.id = b.person_id;
QUERY PLAN
---------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..433.00 rows=1 width=46)
-> Nested Loop (cost=0.00..433.00 rows=1 width=46)
Join Filter: true
-> Table Scan on t_person (cost=0.00..431.00 rows=1 width=19)
-> Index Scan using idex_pd4_pid on t_person_address4 (cost=0.00..2.00 rows=1 width=27)
Index Cond: t_person_address4.person_id = t_person.id
Optimizer status: PQO version 2.65.0
(7 rows)
Time: 12.115 ms
testDB=# explain select * from t_person a join t_person_address2 b on a.id = b.person_id;
QUERY PLAN
-------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice2; segments: 3) (cost=0.00..862.00 rows=1 width=45)
-> Hash Join (cost=0.00..862.00 rows=1 width=45)
Hash Cond: t_person_address2.person_id = t_person.id
-> Redistribute Motion 3:3 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=26)
Hash Key: t_person_address2.person_id
-> Table Scan on t_person_address2 (cost=0.00..431.00 rows=1 width=26)
-> Hash (cost=431.00..431.00 rows=1 width=19)
-> Table Scan on t_person (cost=0.00..431.00 rows=1 width=19)
Optimizer status: PQO version 2.65.0
(9 rows)
Time: 10.453 ms