Flink sql cdc 读取kafka json格式数据

• Flink CDC 读取kafka json格式数据

package com.com.cdc
import org.apache.flink.streaming.api.scala._
import org.apache.flink.table.api.{EnvironmentSettings, Table}
import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment
import org.apache.flink.types.Row
object KafkaJson {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    val bsSettings = EnvironmentSettings.newInstance.useBlinkPlanner.inStreamingMode.build
    // 使用blink创建流式table环境变量
    val sTableEnv = StreamTableEnvironment.create(env, bsSettings)
    // 定义DDL
    val kfkSql =
      s"""
         |CREATE TABLE kafka_table (
         | common ROW(uid string),
         | ts BIGINT
         |) WITH (
         | 'connector' = 'kafka',
         | 'topic' = 'test',
         | 'properties.bootstrap.servers' = 'jeff200:9092',
         | 'format' = 'json',
         | 'scan.startup.mode' = 'earliest-offset'
         |)
         |""".stripMargin
    // 创建虚拟流表
    sTableEnv.executeSql(kfkSql)
    val filterSql =
      s"""
         |SELECT uid, ts
         |FROM kafka_table
         |WHERE uid > 0
       """.stripMargin
    // 执行sql查询
    val table: Table = sTableEnv.sqlQuery(filterSql)
    // 追加流返回DataStream[Row]
    val kafkaStream:DataStream[Row] = sTableEnv.toAppendStream(table)
    kafkaStream.print()

    env.execute("kafka cdc")
  }
}

• 核心依赖

        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-jsonartifactId>
            <version>${flink.version}version>
            <scope>${scope}scope>
        dependency>

        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-table-api-scala-bridge_2.11artifactId>
            <version>${flink.version}version>
            <scope>${scope}scope>
        dependency>
        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-table-planner-blink_2.11artifactId>
            <version>${flink.version}version>
            <scope>${scope}scope>
        dependency>

• kafka test 数据和运行结果

{"ts": 1607003021, "common": {"uid": 1}, "displays": [{"item": 1, "type": "goods"},{"item": 2, "type": "goods"}]}