Flink CDC入门案例

由于Flink CDC是基于日志的方式，因此需要开启MySQL的binlog日志。

开启binlog日志的配置如下

#1.编辑MySQL的配置文件
vim /etc/my.cnf

#添加如下内容
[mysqld]
log-bin=mysql-bin # 开启 binlog
binlog-format=ROW # 选择 ROW 模式
server_id=1 # 配置 MySQL replaction 需要定义，不要和 canal 的 slaveId 重复

#重启MySQL服务
systemctl restart mysqld

MySQL开启binlog后，使用下列查询语句来验证MySQL的binlog是否开启成功：

show variables like '%log_bin%';

数据准备：在MySQL中创建库、表、插入数据

--创建数据库
Create database test character set utf8;

--切换数据库
Use test;

-- 建表
-- 学生表
CREATE TABLE `Student`(
      `s_id` VARCHAR(20),
      `s_name` VARCHAR(20) NOT NULL DEFAULT '',
      `s_birth` VARCHAR(20) NOT NULL DEFAULT '',
      `s_sex` VARCHAR(10) NOT NULL DEFAULT '',
      PRIMARY KEY(`s_id`)
);
-- 成绩表
CREATE TABLE `Score`(
    `s_id` VARCHAR(20),
    `c_id` VARCHAR(20),
    `s_score` INT(3),
    PRIMARY KEY(`s_id`,`c_id`)
);


-- 插入学生表测试数据
insert into Student values('01' , '赵雷' , '1990-01-01' , '男');
insert into Student values('02' , '钱电' , '1990-12-21' , '男');
insert into Student values('03' , '孙风' , '1990-05-20' , '男');
insert into Student values('04' , '李云' , '1990-08-06' , '男');
insert into Student values('05' , '周梅' , '1991-12-01' , '女');
insert into Student values('06' , '吴兰' , '1992-03-01' , '女');
insert into Student values('07' , '郑竹' , '1989-07-01' , '女');
insert into Student values('08' , '王菊' , '1990-01-20' , '女');
-- 成绩表测试数据
insert into Score values('01' , '01' , 80);
insert into Score values('01' , '02' , 90);
insert into Score values('01' , '03' , 99);
insert into Score values('02' , '01' , 70);
insert into Score values('02' , '02' , 60);
insert into Score values('02' , '03' , 80);
insert into Score values('03' , '01' , 80);
insert into Score values('03' , '02' , 80);
insert into Score values('03' , '03' , 80);
insert into Score values('04' , '01' , 50);
insert into Score values('04' , '02' , 30);
insert into Score values('04' , '03' , 20);
insert into Score values('05' , '01' , 76);
insert into Score values('05' , '02' , 87);
insert into Score values('06' , '01' , 31);
insert into Score values('06' , '03' , 34);
insert into Score values('07' , '02' , 89);
insert into Score values('07' , '03' , 98);

启动集群

#0.准备工作
cd $FLINK_HOME/lib，查看是否有如下3个jar包，没有的话需要拷贝进来
commons-cli-1.4.jar
flink-sql-connector-mysql-cdc-2.2.1.jar
flink-sql-parquet_2.12-1.14.5.jar
有的话，表示Flink CDC已经集成了。接下来可以正常登录FlinkSQL客户端。

#1.启动HDFS
start-dfs.sh

#2.启动Flink集群
start-cluster.sh

#3.进入SQL-Client
sql-client.sh

Flink SQL-Client操作

在FlinkSQL中创建映射表

--在FlinkSQL中创建MySQL中Student表的映射表
CREATE TABLE if not exists mysql_cdc_to_test_Student (
     s_id     STRING,
     s_name   STRING,
     s_birth  STRING,
     s_sex    STRING,
     PRIMARY KEY (`s_id`) NOT ENFORCED
) WITH (
    'connector'= 'mysql-cdc',
    'hostname'= '192.168.88.161',
    'port'= '3306',
    'username'= 'root',
    'password'='密码',
    'server-time-zone'= 'Asia/Shanghai',
    'debezium.snapshot.mode'='initial',
    'database-name'= 'test',
    'table-name'= 'Student'
);
--解释
'connector'='mysql-cdc'，指定采用Flink CDC的方式来捕获数据变更
 'server-time-zone'= 'Asia/Shanghai',更换时区为上海时区
'debezium.snapshot.mode'='initial',指定消费模式为initial，表示从历史数据开始消费

---查询语句
select * from mysql_cdc_to_test_Student;