PiflowX组件-PostgresCdc
PostgresCdc组件
组件说明
Postgres CDC连接器允许从PostgreSQL数据库读取快照数据和增量数据。
计算引擎
flink
组件分组
cdc
端口
Inport：默认端口
outport：默认端口
组件属性
| 名称 | 展示名称 | 默认值 | 允许值 | 是否必填 | 描述 | 例子 |
|---|---|---|---|---|---|---|
| hostname | Hostname | “” | 无 | 是 | PostgreSQL数据库服务器的IP地址或主机名。 | 127.0.0.1 |
| username | Username | “” | 无 | 是 | 连接到PostgreSQL数据库服务器时要使用的用户名。 | root |
| password | Password | “” | 无 | 是 | 连接PostgreSQL数据库服务器时使用的密码。 | 123456 |
| databaseName | DatabaseName | “” | 无 | 是 | 要监视的PostgreSQL服务器的数据库名称。 | test |
| schemaName | Schema | “” | 无 | 是 | 要监视的PostgreSQL数据库的Schema。 | public |
| tableName | TableName | “” | 无 | 是 | "需要监视的PostgreSQL数据库的表名。 | test |
| port | Port | 5432 | 无 | 否 | PostgreSQL数据库服务器的整数端口号。 | 5432 |
| slotName | SlotName | “” | 无 | 是 | The name of the PostgreSQL logical decoding slot that was created for streaming changes from a particular plug-in for a particular database/schema. The server uses this slot to stream events to the connector that you are configuring. | |
| Slot names must conform to PostgreSQL replication slot naming rules, which state: “Each replication slot has a name, which can contain lower-case letters, numbers, and the underscore character.” | ||||||
| tableDefinition | TableDefinition | “” | 无 | 是 | Flink table定义。 | |
| properties | PROPERTIES | “” | 无 | 否 | 连接器其他配置。 |
PostgresCdc示例配置
{
"flow":{
"name":"pgcdc",
"uuid":"273553c2ece043c29fba179df6826c5a",
"paths":[
{
"inport":"",
"from":"products",
"to":"SQLQuery",
"outport":""
},
{
"inport":"",
"from":"orders",
"to":"SQLQuery",
"outport":""
},
{
"inport":"",
"from":"shipments",
"to":"SQLQuery",
"outport":""
},
{
"inport":"",
"from":"SQLQuery",
"to":"ShowChangeLogData",
"outport":""
}
],
"engineType":"flink",
"stops":[
{
"name":"shipments",
"bundle":"cn.piflow.bundle.flink.cdc.postgres.PostgresCdc",
"uuid":"13ac002a8a4d404e8dda77c9a1d8360f",
"properties":{
"hostname":"192.168.186.102",
"username":"postgres",
"port":"5432",
"properties":{
},
"schemaName":"public",
"tableName":"shipments",
"tableDefinition":{
"tableBaseInfo":{
"ifNotExists":true,
"registerTableName":"shipments"
},
"physicalColumnDefinition":[
{
"columnName":"shipment_id",
"columnType":"INT",
"primaryKey":true
},
{
"columnName":"order_id",
"columnType":"INT"
},
{
"columnName":"origin",
"columnType":"STRING"
},
{
"columnName":"destination",
"columnType":"STRING"
},
{
"columnName":"is_arrived",
"columnType":"BOOLEAN"
}
],
"asSelectStatement":{
},
"likeStatement":{
}
},
"databaseName":"postgres",
"slotName":"flink",
"password":"postgres"
},
"customizedProperties":{
}
},
{
"name":"SQLQuery",
"bundle":"cn.piflow.bundle.flink.common.SQLQuery",
"uuid":"a5afa1368e1e4e348950bb2eda2011a8",
"properties":{
"viewName":"temp",
"sql":"SELECT\n o.*,\n p.name,\n p.description,\n s.shipment_id,\n s.origin,\n s.destination,\n s.is_arrived\nFROM\n orders AS o\n LEFT JOIN products AS p ON o.product_id = p.id\n LEFT JOIN shipments AS s ON o.order_id = s.order_id"
},
"customizedProperties":{
}
},
{
"name":"ShowChangeLogData",
"bundle":"cn.piflow.bundle.flink.common.ShowChangeLogData",
"uuid":"c6604e600aa645d29264f4be5ce4e2eb",
"properties":{
"showNumber":"10"
},
"customizedProperties":{
}
},
{
"name":"products",
"bundle":"cn.piflow.bundle.flink.cdc.mysql.MysqlCdc",
"uuid":"ab91c42f26cb4e119e34830178611293",
"properties":{
"hostname":"192.168.186.102",
"username":"root",
"serverId":"5400",
"port":"3306",
"properties":{
"server-time-zone":"UTC"
},
"tableName":"products",
"tableDefinition":{
"tableBaseInfo":{
"registerTableName":"products"
},
"physicalColumnDefinition":[
{
"columnName":"id",
"columnType":"INT",
"primaryKey":true
},
{
"columnName":"name",
"columnType":"STRING"
},
{
"columnName":"description",
"columnType":"STRING"
}
],
"asSelectStatement":{
},
"likeStatement":{
}
},
"databaseName":"mydb",
"password":"123456"
},
"customizedProperties":{
}
},
{
"name":"orders",
"bundle":"cn.piflow.bundle.flink.cdc.mysql.MysqlCdc",
"uuid":"4ffa58f3db6144739f6b797fd3839025",
"properties":{
"hostname":"192.168.186.102",
"username":"root",
"serverId":"5410",
"port":"3306",
"properties":{
"scan.incremental.snapshot.chunk.key-column":"order_id",
"server-time-zone":"UTC"
},
"tableName":"orders",
"tableDefinition":{
"tableBaseInfo":{
"ifNotExists":true,
"registerTableName":"orders"
},
"physicalColumnDefinition":[
{
"columnName":"order_id",
"columnType":"INT"
},
{
"columnName":"order_date",
"columnType":"TIMESTAMP",
"length":0
},
{
"columnName":"customer_name",
"columnType":"STRING"
},
{
"columnName":"price",
"columnType":"DECIMAL",
"precision":10,
"scale":5
},
{
"columnName":"product_id",
"columnType":"INT"
},
{
"columnName":"order_status",
"columnType":"BOOLEAN"
}
],
"asSelectStatement":{
},
"likeStatement":{
}
},
"databaseName":"mydb",
"password":"123456"
},
"customizedProperties":{
}
}
]
}
}
示例说明
本示例演示了基于PiflowX构建MySQL和Postgres的Streaming ETL。
演示DEMO
基于 PiflowX构建 MySQL 和 Postgres 的 Streaming ETL