Apache Paimon-并发写
真实业务场景
假设有一张很大的表，字段很多，单个job写的很慢，这个时候要并发写。paimon 基于 ‘merge-engine’ = 'partial-update’和Sequence Group可以支持并发写，多个job写不同的字段，每个job维护自己的Sequence Group
模拟场景
由于本地环境的问题，暂时的模拟是启动三个程序消费三个socket流，三个socket流是同一份数据，然后写入同一张表
创建Paimon表
CREATE TABLE if not exists MyTable(
id INT ,
col1_1 STRING,
col1_2 STRING,
col1_3 STRING,
col1_4 STRING,
version1 INT ,
col2_1 STRING,
col2_2 STRING,
col2_3 STRING,
col2_4 STRING,
version2 INT,
col3_1 STRING,
col3_2 STRING,
col3_3 STRING,
col3_4 STRING,
version3 INT ,
PRIMARY KEY (id) NOT ENFORCED
)
WITH (
'merge-engine'='partial-update',
'fields.version1.sequence-group'='col1_1,col1_2,col1_3,col1_4',
'fields.version2.sequence-group'='col2_1,col2_2,col2_3,col2_4',
'fields.version3.sequence-group'='col3_1,col3_2,col3_3,col3_4',
'bucket' = '2'
);
数据准备
{"id":1000,"col1_1":"col1_1-0","col1_2":"col1_2-0","col1_3":"col1_3-0","col1_4":"col1_4-0","version1":0,"col2_1":"col2_1-0","col2_2":"col2_2-0","col2_3":"col2_3-0","col2_4":"col2_4-0","version2":0,"col3_1":"col3_1-0","col3_2":"col3_2-0","col3_3":"col3_3-0","col3_4":"col3_4-0","version3":0}
{"id":1000,"col1_1":"col1_1-0","col1_2":"dddd","col1_3":"bbbb","col1_4":"col1_4-0","version1":1,"col2_1":"qqqq-0","col2_2":"cccc","col2_3":"col2_3-0","col2_4":"col2_4-0","version2":2,"col3_1":"col3_1-0","col3_2":"col3_2-0","col3_3":"tyty-0","col3_4":"aaaa","version3":3
DML语句准备
//三个程序分别执行下面三个语句
insert into MyTable
(id,
col1_1,
col1_2,
col1_3,
col1_4,
version1,
col2_1,
col2_2,
col2_3,
col2_4,
version2,
col3_1,
col3_2,
col3_3,
col3_4,
version3)
select
id,
col1_1,
col1_2,
col1_3,
col1_4,
version1,
CAST(NULL AS String)as col2_1,
CAST(NULL AS String)as col2_2,
CAST(NULL AS String)as col2_3,
CAST(NULL AS String)as col2_4,
CAST(NULL AS INT)as version2,
CAST(NULL AS String)as col3_1,
CAST(NULL AS String)as col3_2,
CAST(NULL AS String)as col3_3,
CAST(NULL AS String)as col3_4,
CAST(NULL AS INT)as version3
from
InputTable;
//
insert into MyTable (id ,
col1_1 ,
col1_2 ,
col1_3 ,
col1_4 ,
version1 ,
col2_1 ,
col2_2 ,
col2_3 ,
col2_4 ,
version2 ,
col3_1 ,
col3_2 ,
col3_3 ,
col3_4 ,
version3) select
id ,
CAST(NULL AS String) as col1_1 ,
CAST(NULL AS String) as col1_2 ,
CAST(NULL AS String) as col1_3 ,
CAST(NULL AS String) as col1_4 ,
CAST(NULL AS INT) version1 ,
col2_1 ,
col2_2 ,
col2_3 ,
col2_4 ,
version2 ,
CAST(NULL AS String) as col3_1 ,
CAST(NULL AS String) as col3_2 ,
CAST(NULL AS String) as col3_3 ,
CAST(NULL AS String) as col3_4 ,
CAST(NULL AS INT) as version3
from
InputTable;
//
insert into MyTable (id ,
col1_1 ,
col1_2 ,
col1_3 ,
col1_4 ,
version1 ,
col2_1 ,
col2_2 ,
col2_3 ,
col2_4 ,
version2 ,
col3_1 ,
col3_2 ,
col3_3 ,
col3_4 ,
version3) select
id ,
CAST(NULL AS String) as col1_1 ,
CAST(NULL AS String) as col1_2 ,
CAST(NULL AS String) as col1_3 ,
CAST(NULL AS String) as col1_4 ,
CAST(NULL AS INT) version1 ,
CAST(NULL AS String) as col2_1 ,
CAST(NULL AS String) as col2_2 ,
CAST(NULL AS String) as col2_3 ,
CAST(NULL AS String) as col2_4 ,
CAST(NULL AS INT) as version2 ,
col3_1 ,
col3_2 ,
col3_3 ,
col3_4 ,
version3
from
InputTable;
flink 代码
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestOptions;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.Schema;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.types.DataType;
import org.apache.flink.types.Row;
import java.util.List;
public class Sink2Paimon {
public static void writeTo() {
Configuration config = new Configuration();
// 指定WebUI界面的访问端口，默认是8081
config.setString(RestOptions.BIND_PORT, "9999");
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
env.enableCheckpointing(5000, CheckpointingMode.EXACTLY_ONCE);
DataStream<String> socketStream = env.socketTextStream("localhost", 9000);
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
Schema schema1 = Schema.newBuilder()
.column("id", DataTypes.INT())
.column("col1_1", DataTypes.STRING())
.column("col1_2", DataTypes.STRING())
.column("col1_3", DataTypes.STRING())
.column("col1_4", DataTypes.STRING())
.column("version1", DataTypes.INT())
.column("col2_1", DataTypes.STRING())
.column("col2_2", DataTypes.STRING())
.column("col2_3", DataTypes.STRING())
.column("col2_4", DataTypes.STRING())
.column("version2", DataTypes.INT())
.column("col3_1", DataTypes.STRING())
.column("col3_2", DataTypes.STRING())
.column("col3_3", DataTypes.STRING())
.column("col3_4", DataTypes.STRING())
.column("version3", DataTypes.INT())
.build();
SingleOutputStreamOperator<Row> returns = socketStream.map(new MapFunction<String, Row>() {
@Override
public Row map(String value) throws Exception {
Schema aaa = Schema.newBuilder()
.column("id", DataTypes.INT())
.column("col1_1", DataTypes.STRING())
.column("col1_2", DataTypes.STRING())
.column("col1_3", DataTypes.STRING())
.column("col1_4", DataTypes.STRING())
.column("version1", DataTypes.INT())
.column("col2_1", DataTypes.STRING())
.column("col2_2", DataTypes.STRING())
.column("col2_3", DataTypes.STRING())
.column("col2_4", DataTypes.STRING())
.column("version2", DataTypes.INT())
.column("col3_1", DataTypes.STRING())
.column("col3_2", DataTypes.STRING())
.column("col3_3", DataTypes.STRING())
.column("col3_4", DataTypes.STRING())
.column("version3", DataTypes.INT())
.build();
JSONObject jsonObject = JSON.parseObject(value);
List<Schema.UnresolvedColumn> columns = aaa.getColumns();
Row row = new Row(columns.size());
for (int i = 0; i < columns.size(); i++) {
String name = columns.get(i).getName();
row.setField(i, jsonObject.get(name));
}
return row;
}
}).returns(Types.ROW_NAMED(
new String[]{"id",
"col1_1",
"col1_2",
"col1_3",
"col1_4",
"version1",
"col2_1",
"col2_2",
"col2_3",
"col2_4",
"version2",
"col3_1",
"col3_2",
"col3_3",
"col3_4",
"version3"},
Types.INT, Types.STRING, Types.STRING, Types.STRING,
Types.STRING, Types.INT, Types.STRING, Types.STRING,
Types.STRING, Types.STRING, Types.INT, Types.STRING, Types.STRING, Types.STRING, Types.STRING, Types.INT
));
Table table = tableEnv.fromDataStream(returns, schema1);
tableEnv.executeSql("CREATE CATALOG paimon WITH ('type' = 'paimon', 'warehouse'='file:///D://data')");
tableEnv.executeSql("USE CATALOG paimon");
tableEnv.createTemporaryView("InputTable", table);
tableEnv.executeSql("CREATE TABLE if not exists MyTable(\n" +
"id INT ,\n" +
"col1_1 STRING,\n" +
"col1_2 STRING,\n" +
"col1_3 STRING,\n" +
"col1_4 STRING,\n" +
"version1 INT ,\n" +
"col2_1 STRING,\n" +
"col2_2 STRING,\n" +
"col2_3 STRING,\n" +
"col2_4 STRING,\n" +
"version2 INT,\n" +
"col3_1 STRING,\n" +
"col3_2 STRING,\n" +
"col3_3 STRING,\n" +
"col3_4 STRING,\n" +
"version3 INT ,\n" +
"PRIMARY KEY (id) NOT ENFORCED\n" +
")\n" +
"WITH (\n" +
"'merge-engine'='partial-update',\n" +
"'fields.version1.sequence-group'='col1_1,col1_2,col1_3,col1_4',\n" +
"'fields.version2.sequence-group'='col2_1,col2_2,col2_3,col2_4',\n" +
"'fields.version3.sequence-group'='col3_1,col3_2,col3_3,col3_4',\n" +
"'bucket' = '2'\n" +
")");
//1
tableEnv.executeSql(
"insert into MyTable\n" +
"(id,\n" +
"col1_1,\n" +
"col1_2,\n" +
"col1_3,\n" +
"col1_4,\n" +
"version1,\n" +
"col2_1,\n" +
"col2_2,\n" +
"col2_3,\n" +
"col2_4,\n" +
"version2,\n" +
"col3_1,\n" +
"col3_2,\n" +
"col3_3,\n" +
"col3_4,\n" +
"version3)\n" +
"select\n" +
"id,\n" +
"col1_1,\n" +
"col1_2,\n" +
"col1_3,\n" +
"col1_4,\n" +
"version1,\n" +
"CAST(NULL AS String)as col2_1,\n" +
"CAST(NULL AS String)as col2_2,\n" +
"CAST(NULL AS String)as col2_3,\n" +
"CAST(NULL AS String)as col2_4,\n" +
"CAST(NULL AS INT)as version2,\n" +
"CAST(NULL AS String)as col3_1,\n" +
"CAST(NULL AS String)as col3_2,\n" +
"CAST(NULL AS String)as col3_3,\n" +
"CAST(NULL AS String)as col3_4,\n" +
"CAST(NULL AS INT)as version3\n" +
"from\n" +
"InputTable");
}
public static void main(String[] args) {
writeTo();
}
}
执行结果
//{"id":1000,"col1_1":"col1_1-0","col1_2":"col1_2-0","col1_3":"col1_3-0","col1_4":"col1_4-0","version1":0,"col2_1":"col2_1-0","col2_2":"col2_2-0","col2_3":"col2_3-0","col2_4":"col2_4-0","version2":0,"col3_1":"col3_1-0","col3_2":"col3_2-0","col3_3":"col3_3-0","col3_4":"col3_4-0","version3":0}
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
| id | col1_1 | col1_2 | col1_3 | col1_4 | version1 | col2_1 | col2_2 | col2_3 | col2_4 | version2 | col3_1 | col3_2 | col3_3 | col3_4 | version3 |
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
| 1000 | col1_1-0 | col1_2-0 | col1_3-0 | col1_4-0 | 0 | col2_1-0 |col2_2-0 | col2_3-0 | col2_4-0 | 0 | col3_1-0 | col3_2-0 |col3_3-0 | col3_4-0 | 0 |
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
//{"id":1000,"col1_1":"col1_1-0","col1_2":"dddd","col1_3":"bbbb","col1_4":"col1_4-0","version1":1,"col2_1":"qqqq-0","col2_2":"cccc","col2_3":"col2_3-0","col2_4":"col2_4-0","version2":2,"col3_1":"col3_1-0","col3_2":"col3_2-0","col3_3":"tyty-0","col3_4":"aaaa","version3":3}
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
| id | col1_1 | col1_2 | col1_3 | col1_4 | version1 | col2_1 | col2_2 | col2_3 | col2_4 | version2 | col3_1 | col3_2 | col3_3 | col3_4 | version3 |
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
| 1000 | col1_1-0 | dddd | bbbb | col1_4-0 | 1 | qqqq-0 | cccc | col2_3-0 | col2_4-0 | 2 | col3_1-0 | col3_2-0 | tyty-0 | aaaa | 3 |
+------+-----------+-----------+-----------+------------+-------------+----------+---------+-------------+-----------+-------------+----------+----------+---------+----------+-------------+
结论
从结果来看paimon对于并发写是没问题的，上面只是两条数据简单的测了下，后于有机会的话会再试下