第三章 数据采集模块之FlinkCDC实时采集Mysql数据

1、Mysql数据准备

（1）创建实时同步数据库

create database flink_gmall

（2）将Mysql.sql文件导入到Mysql中

source /opt/data/Mysql.sql

（3）查看数据库表

show tables;

2、开启数据库的binlog

（1）在mysql中对需要进行实时数据监测的库开启binlog

sudo vim /etc/my.cnf

#添加数据库的binlog
server-id=1
log-bin=mysql-bin
binlog_format=row
binlog-do-db=flink_gmall


#重启MySQL服务
sudo systemctl restart mysqld

（2）查询生成日志

cd /var/lib/mysql

3、编写脚本

（1）全量写入后增量检测

package com.lhw;

import com.ververica.cdc.connectors.mysql.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.ververica.cdc.debezium.DebeziumSourceFunction;
import com.ververica.cdc.debezium.StringDebeziumDeserializationSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class MysqlFlinkCDC {
    public static void main(String[] args) throws Exception {
        // 1、获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(2);

        // 2、通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> mysqlSource = MySqlSource.<String>builder()
                .hostname("hadoop102")
                .port(3306)
                .databaseList("flink_gmall")  // set captured database
                .tableList("flink_gmall.base_category1") // 如果不添加该参数,则消费指定数据库中的所有表
                .username("root")
                .password("123456")
                /**initial:初始化快照,即全量导入后增量导入(检测更新数据写入)
                 * latest:只进行增量导入(不读取历史变化)
                 * timestamp:指定时间戳进行数据导入(大于等于指定时间错读取数据)
                 */
                .startupOptions(StartupOptions.initial())
                .deserializer(new StringDebeziumDeserializationSchema())
                .build();
        // 3、使用CDC Source方式从mysql中读取数据
        DataStreamSource<String> mysqlDS = env.addSource(mysqlSource);

        // 4、打印数据
        mysqlDS.print();

        // 5、执行任务
        env.execute("flinkcdcmysql");
    }
}

（2）运行结果

• mysql源表数据
  

• flinkcdc读取历史变化数据

（3）检测变化

• mysql源表新增&修改数据

#新增数据
insert  into `base_category1`(`id`,`name`) values(18,'珠宝首饰');

#更新数据
update base_category1 set name='奢侈品' where id=18;

• 检测变化结果

R代表历史,C代编新增,U代码更新

4、集群部署&测试

（1）脚本修改

package com.lhw;

import com.ververica.cdc.connectors.mysql.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.ververica.cdc.debezium.DebeziumSourceFunction;
import com.ververica.cdc.debezium.StringDebeziumDeserializationSchema;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class MysqlFlinkCDC {
    public static void main(String[] args) throws Exception {
        // 1、获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(2);

        // 2、开启CK并指定状态后端为FS
        // 2.1、制定存储ck地址
        env.setStateBackend(new FsStateBackend("hdfs://hadoop102:8020/flinkCDC"));
        // 2.2、指定ck储存触发间隔时间
        env.enableCheckpointing(5000L);
        // 2.3、指定ck模式
        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        // 2.4、指定超时时间
        env.getCheckpointConfig().setAlignmentTimeout(10000);
        // 2.5、CK最小触发间隔时间
        env.getCheckpointConfig().setMaxConcurrentCheckpoints(2000);

        // 3、通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> mysqlSource = MySqlSource.<String>builder()
                .hostname("hadoop102")
                .port(3306)
                .databaseList("flink_gmall")  // set captured database
                .tableList("flink_gmall.base_category1") // 如果不添加该参数,则消费指定数据库中的所有表
                .username("root")
                .password("123456")
                /**initial初始化快照,即全量导入后增量导入(检测更新数据写入)
                 * latest:只进行增量导入(不读取历史变化)
                 * timestamp:指定时间戳进行数据导入(大于等于指定时间错读取数据)
                 */
                .startupOptions(StartupOptions.initial())
                .deserializer(new StringDebeziumDeserializationSchema())
                .build();
        // 4、使用CDC Source方式从mysql中读取数据
        DataStreamSource<String> mysqlDS = env.addSource(mysqlSource);

        // 5、打印数据
        mysqlDS.print();

        // 5、执行任务
        env.execute("flinkcdcmysql");
    }
}

（2）打包上传

• 打包：IDEA–>Maven–>package

• 上传到服务器上

rz gmall-flinkcdc-mysql-1.0-SNAPSHOT-jar-with-dependencies.jar

（3）部署测试

• 开启Flink

• 运行脚本

bin/flink run -m hadoop102:8081 -c com.lhw.MysqlFlinkCDC /opt/data/gmall-flinkcdc-mysql.jar

• 查看WEB界面

• 查看Job输出

• 查看CK信息

（4）Flink运行错误问题

• 问题现状：java.lang.NoSuchMethodError: org.apache.flink.streaming.api.environment.CheckpointConfig.setAlignmentTimeout(J)V

java.lang.NoSuchMethodError: org.apache.flink.streaming.api.environment.CheckpointConfig.setAlignmentTimeout(J)V
	at com.lhw.MysqlFlinkCDC.main(MysqlFlinkCDC.java:26)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114)
	at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:812)
	at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:246)
	at org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1054)
	at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1132)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)
	at org.apache.flink.runtime.security.contexts.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
	at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1132)

• 问题原因

pom文件中flink版本为1.12.0,服务器上的flink版本为1.13.1

• 解决方式

将pom文件中的版本替换为测试环境中的版本

5、断点续传测试

• 模仿FlinkCDC任务挂掉重启后的执行情况

（1）手动保存CK

bin/flink savepoint cd4273e86cc0cc82a1b8b374ae9f7bdf hdfs://hadoop102:8020/flinkCDC/gmall-flinkcdc/sv

（2）重启Flink，模拟任务done掉

bin/stop-cluster.sh

（3）在mysql源表中新增数据

insert  into `base_category1`(`id`,`name`) values(19,'珠宝首饰');

insert  into `base_category1`(`id`,`name`) values(20,'轻奢品');

（4）重启命令

bin/flink run -m hadoop102:8081 -s hdfs://hadoop102:8020/flinkCDC/gmall-flinkcdc/sv/savepoint-cd4273-0795c5a912b4 -c com.lhw.MysqlFlinkCDC /opt/data/gmall-flinkcdc-mysql2.jar

• 在WEB端查看运行结果

暂停

6、自定义反序列化器

• 固定格式显示

package com.lhw;

import com.alibaba.fastjson.JSONObject;
import com.ververica.cdc.debezium.DebeziumDeserializationSchema;
import io.debezium.data.Envelope;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.util.Collector;
import org.apache.kafka.connect.data.Field;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord;

import java.util.List;

public class CustomerDeserialization implements DebeziumDeserializationSchema<String> {

    /**
     * 封装的数据格式
     * {
     * "database":"",
     * "tableName":"",
     * "before":{"id":"","tm_name":""....},
     * "after":{"id":"","tm_name":""....},
     * "type":"c u d",
     * //"ts":156456135615
     * }
     */
    @Override
    public void deserialize(SourceRecord sourceRecord, Collector<String> collector) throws Exception {

        //1.创建JSON对象用于存储最终数据
        JSONObject result = new JSONObject();

        //2.获取库名&表名
        String topic = sourceRecord.topic();
        String[] fields = topic.split("\\.");
        String database = fields[1];
        String tableName = fields[2];

        Struct value = (Struct) sourceRecord.value();
        //3.获取"before"数据
        Struct before = value.getStruct("before");
        JSONObject beforeJson = new JSONObject();
        if (before != null) {
            Schema beforeSchema = before.schema();
            List<Field> beforeFields = beforeSchema.fields();
            for (Field field : beforeFields) {
                Object beforeValue = before.get(field);
                beforeJson.put(field.name(), beforeValue);
            }
        }

        //4.获取"after"数据
        Struct after = value.getStruct("after");
        JSONObject afterJson = new JSONObject();
        if (after != null) {
            Schema afterSchema = after.schema();
            List<Field> afterFields = afterSchema.fields();
            for (Field field : afterFields) {
                Object afterValue = after.get(field);
                afterJson.put(field.name(), afterValue);
            }
        }

        //5.获取操作类型  CREATE UPDATE DELETE
        Envelope.Operation operation = Envelope.operationFor(sourceRecord);
        String type = operation.toString().toLowerCase();
        if ("create".equals(type)) {
            type = "insert";
        }

        //6.将字段写入JSON对象
        result.put("database", database);
        result.put("tableName", tableName);
        result.put("before", beforeJson);
        result.put("after", afterJson);
        result.put("type", type);

        //7.输出数据
        collector.collect(result.toJSONString());

    }

    @Override
    public TypeInformation<String> getProducedType() {
        return BasicTypeInfo.STRING_TYPE_INFO;
    }
}

• 运行结果
  

7、FlinkSQLAPI编码测试

（1）编写代码

package com.lhw;

import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;

public class MysqlFlinkSqlCDC {
    public static void main(String[] args) throws Exception {
        // 1、创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);

        // 2、创建Flink-Mysql-CDC的Source
        tableEnv.executeSql(
                "CREATE TABLE user_info (" +
                        " id INT," +
                        " name STRING," +
                        " PRIMARY KEY (`id`)  NOT ENFORCED" +
                        ") WITH (" +
                        " 'connector' = 'mysql-cdc'," +
                        " 'hostname' = '192.168.6.102'," +
                        " 'port' = '3306'," +
                        " 'username' = 'root'," +
                        " 'password' = '123456'," +
                        " 'database-name' = 'flink_gmall'," +
                        " 'table-name' = 'base_category1'," +
                        "   'debezium.snapshot.mode' = 'initial' " +  // 读取mysql的全量,增量以及更新数据
                        ")");
        // 3、打印输出
        Table table = tableEnv.sqlQuery("select * from user_info");
        DataStream<Tuple2<Boolean, Row>> stream = tableEnv.toRetractStream(table,Row.class);
        stream.print();

        // 4、提交任务
        env.execute("Flinksql");
    }
}

（2）查看运行结果

（3）在源表中增加数据

insert  into `base_category1`(`id`,`name`) values(19,'珠宝首饰');

（4）查看监控变化结果

8、FlinkSQL和FlinkStream比对

（1）DataStream

• 优点：多表多库
• 缺点：需要自定义反序列化器

（2）FlinkSQL

• 优点：不需要自定义反序列化器
• 缺点：单表查询