flink table 使用Kafka Connector处理复杂json

  flink kafka connector处理复杂json字符串,直接上代码。

StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build();
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,bsSettings);
        tableEnv.connect(new Kafka()
                .version("universal")
                .topic("w001")
                .property("zookeeper.connect", "192.168.0.1:2181")
                .property("bootstrap.servers", "192.168.0.1:9092,192.168.0.2:9092,192.168.0.3:9092")
                .property("group.id", "w01")
                .startFromLatest()
        )
                .withFormat(new Json().deriveSchema())  //jsonSchema(JSON_SCHEMA).failOnMissingField(false)
//{"id":1553650604,"code":200,"results":{"id":1000,"items":[{"name":"renhao","title":"flink"}],"data":{"val":"test","lists":[{"sex":"男"}]}}}
                .withSchema(new Schema().field("id", Types.LONG).field("code", Types.INT)
                        .field("results",Types.ROW_NAMED(new String[]{"id","items","data"},new TypeInformation[]{
                        Types.LONG,
                               ObjectArrayTypeInfo.getInfoFor(Row[].class,
                                        Types.ROW_NAMED(
                                                new String[]{"name", "title"},
                                                new TypeInformation[]{Types.STRING,Types.STRING})
                                ),
                                Types.ROW_NAMED(new String[]{"val","lists"},new TypeInformation[]{Types.STRING, ObjectArrayTypeInfo.getInfoFor(Row[].class,
                                        Types.ROW_NAMED(
                                                new String[]{"sex"},
                                                new TypeInformation[]{Types.STRING})
                                )})

                        })))
                .inAppendMode()
                .registerTableSource("test");
        Table query = tableEnv.sqlQuery("select id,code,results.id as rid ,items[1].name as name ,results.data.val as data ,lists[1].sex as sex from test");
        tableEnv.toAppendStream(query, Row.class).print();
        tableEnv.execute("streaming");

 

你可能感兴趣的:(flink,个人总结)