otter-harbor同步
一. 部署及依赖
otter Github
(一). 服务启动
1. mysql
- 5.6版本以上，作为
otter-manger使用的数据库
# mysql
docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:5.7 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci;
(1). mysql初始化
SQL
otter manager用的数据库, 我选用的是 4.2.18 版本的 otter-manager, 不同版本不同的sql
# 可通过docker cp的方式挂到容器中
otter-manager-init.sql
CREATE DATABASE /*!32312 IF NOT EXISTS*/ `otter` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */;
USE `otter`;
SET sql_mode='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
CREATE TABLE `ALARM_RULE` (
`ID` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`MONITOR_NAME` varchar(1024) DEFAULT NULL,
`RECEIVER_KEY` varchar(1024) DEFAULT NULL,
`STATUS` varchar(32) DEFAULT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`DESCRIPTION` varchar(256) DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`MATCH_VALUE` varchar(1024) DEFAULT NULL,
`PARAMETERS` text DEFAULT NULL,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `AUTOKEEPER_CLUSTER` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`CLUSTER_NAME` varchar(200) NOT NULL,
`SERVER_LIST` varchar(1024) NOT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `CANAL` (
`ID` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) DEFAULT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`PARAMETERS` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `CANALUNIQUE` (`NAME`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `CHANNEL` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) NOT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`PARAMETERS` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `CHANNELUNIQUE` (`NAME`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `COLUMN_PAIR` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`SOURCE_COLUMN` varchar(200) DEFAULT NULL,
`TARGET_COLUMN` varchar(200) DEFAULT NULL,
`DATA_MEDIA_PAIR_ID` bigint(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_DATA_MEDIA_PAIR_ID` (`DATA_MEDIA_PAIR_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `COLUMN_PAIR_GROUP` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`DATA_MEDIA_PAIR_ID` bigint(20) NOT NULL,
`COLUMN_PAIR_CONTENT` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_DATA_MEDIA_PAIR_ID` (`DATA_MEDIA_PAIR_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `DATA_MEDIA` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) NOT NULL,
`NAMESPACE` varchar(200) NOT NULL,
`PROPERTIES` varchar(1000) NOT NULL,
`DATA_MEDIA_SOURCE_ID` bigint(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `DATAMEDIAUNIQUE` (`NAME`,`NAMESPACE`,`DATA_MEDIA_SOURCE_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `DATA_MEDIA_PAIR` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`PULLWEIGHT` bigint(20) DEFAULT NULL,
`PUSHWEIGHT` bigint(20) DEFAULT NULL,
`RESOLVER` text DEFAULT NULL,
`FILTER` text DEFAULT NULL,
`SOURCE_DATA_MEDIA_ID` bigint(20) DEFAULT NULL,
`TARGET_DATA_MEDIA_ID` bigint(20) DEFAULT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`COLUMN_PAIR_MODE` varchar(20) DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_PipelineID` (`PIPELINE_ID`,`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `DATA_MEDIA_SOURCE` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) NOT NULL,
`TYPE` varchar(20) NOT NULL,
`PROPERTIES` varchar(1000) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `DATAMEDIASOURCEUNIQUE` (`NAME`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `DELAY_STAT` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`DELAY_TIME` bigint(20) NOT NULL,
`DELAY_NUMBER` bigint(20) NOT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_PipelineID_GmtModified_ID` (`PIPELINE_ID`,`GMT_MODIFIED`,`ID`),
KEY `idx_Pipeline_GmtCreate` (`PIPELINE_ID`,`GMT_CREATE`),
KEY `idx_GmtCreate_id` (`GMT_CREATE`,`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `LOG_RECORD` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NID` varchar(200) DEFAULT NULL,
`CHANNEL_ID` varchar(200) NOT NULL,
`PIPELINE_ID` varchar(200) NOT NULL,
`TITLE` varchar(1000) DEFAULT NULL,
`MESSAGE` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `logRecord_pipelineId` (`PIPELINE_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `NODE` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) NOT NULL,
`IP` varchar(200) NOT NULL,
`PORT` bigint(20) NOT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`PARAMETERS` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `NODEUNIQUE` (`NAME`,`IP`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `PIPELINE` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NAME` varchar(200) NOT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`PARAMETERS` text DEFAULT NULL,
`CHANNEL_ID` bigint(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `PIPELINEUNIQUE` (`NAME`,`CHANNEL_ID`),
KEY `idx_ChannelID` (`CHANNEL_ID`,`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `PIPELINE_NODE_RELATION` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`NODE_ID` bigint(20) NOT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`LOCATION` varchar(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_PipelineID` (`PIPELINE_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `SYSTEM_PARAMETER` (
`ID` bigint(20) unsigned NOT NULL,
`VALUE` text DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `TABLE_HISTORY_STAT` (
`ID` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`FILE_SIZE` bigint(20) DEFAULT NULL,
`FILE_COUNT` bigint(20) DEFAULT NULL,
`INSERT_COUNT` bigint(20) DEFAULT NULL,
`UPDATE_COUNT` bigint(20) DEFAULT NULL,
`DELETE_COUNT` bigint(20) DEFAULT NULL,
`DATA_MEDIA_PAIR_ID` bigint(20) DEFAULT NULL,
`PIPELINE_ID` bigint(20) DEFAULT NULL,
`START_TIME` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`END_TIME` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_DATA_MEDIA_PAIR_ID_END_TIME` (`DATA_MEDIA_PAIR_ID`,`END_TIME`),
KEY `idx_GmtCreate_id` (`GMT_CREATE`,`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `TABLE_STAT` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`FILE_SIZE` bigint(20) NOT NULL,
`FILE_COUNT` bigint(20) NOT NULL,
`INSERT_COUNT` bigint(20) NOT NULL,
`UPDATE_COUNT` bigint(20) NOT NULL,
`DELETE_COUNT` bigint(20) NOT NULL,
`DATA_MEDIA_PAIR_ID` bigint(20) NOT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_PipelineID_DataMediaPairID` (`PIPELINE_ID`,`DATA_MEDIA_PAIR_ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `THROUGHPUT_STAT` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`TYPE` varchar(20) NOT NULL,
`NUMBER` bigint(20) NOT NULL,
`SIZE` bigint(20) NOT NULL,
`PIPELINE_ID` bigint(20) NOT NULL,
`START_TIME` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`END_TIME` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `idx_PipelineID_Type_GmtCreate_ID` (`PIPELINE_ID`,`TYPE`,`GMT_CREATE`,`ID`),
KEY `idx_PipelineID_Type_EndTime_ID` (`PIPELINE_ID`,`TYPE`,`END_TIME`,`ID`),
KEY `idx_GmtCreate_id` (`GMT_CREATE`,`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `USER` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`USERNAME` varchar(20) NOT NULL,
`PASSWORD` varchar(20) NOT NULL,
`AUTHORIZETYPE` varchar(20) NOT NULL,
`DEPARTMENT` varchar(20) NOT NULL,
`REALNAME` varchar(20) NOT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
UNIQUE KEY `USERUNIQUE` (`USERNAME`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE `DATA_MATRIX` (
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
`GROUP_KEY` varchar(200) DEFAULT NULL,
`MASTER` varchar(200) DEFAULT NULL,
`SLAVE` varchar(200) DEFAULT NULL,
`DESCRIPTION` varchar(200) DEFAULT NULL,
`GMT_CREATE` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
`GMT_MODIFIED` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ID`),
KEY `GROUPKEY` (`GROUP_KEY`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE IF NOT EXISTS `meta_history` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
`sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
`sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
`sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
`sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表';
CREATE TABLE IF NOT EXISTS `meta_snapshot` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键',
`gmt_create` datetime NOT NULL COMMENT '创建时间',
`gmt_modified` datetime NOT NULL COMMENT '修改时间',
`destination` varchar(128) DEFAULT NULL COMMENT '通道名称',
`binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名',
`binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量',
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
`data` longtext DEFAULT NULL COMMENT '表结构数据',
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
PRIMARY KEY (`id`),
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
KEY `destination` (`destination`),
KEY `destination_timestamp` (`destination`,`binlog_timestamp`),
KEY `gmt_modified` (`gmt_modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表';
insert into USER(ID,USERNAME,PASSWORD,AUTHORIZETYPE,DEPARTMENT,REALNAME,GMT_CREATE,GMT_MODIFIED) values(null,'admin','801fc357a5a74743894a','ADMIN','admin','admin',now(),now());
insert into USER(ID,USERNAME,PASSWORD,AUTHORIZETYPE,DEPARTMENT,REALNAME,GMT_CREATE,GMT_MODIFIED) values(null,'guest','471e02a154a2121dc577','OPERATOR','guest','guest',now(),now());
2. zookeeper
# zk, 不用单独配置zk,集群的 单机的都可以
docker run -d -p 2181:2181 --name=zookeeper --privileged zookeeper
3. otter-manager
- 公网可拉取到镜像:
aeert/otter-manager:4.2.18, 因每个人的访问IP或者域名不同, 需要对镜像做修改, - 修改端口在启动命令中加入了:
sed -i 's/otter.port = 8080/otter.port = '$PORT'/' conf/otter.properties &&- 如果要基于域名访问, 也需要修改
domainName
FROM aeert/otter-manager:4.2.18
ENTRYPOINT ["/bin/sh", "-c", "sed -i 's/domainName = 127.0.0.1/domainName = '$IP'/' conf/otter.properties && sed -i 's/otter.port = 8080/otter.port = '$PORT'/' conf/otter.properties && sed -i 's/127.0.0.1:3306/'$DB_IP'/' conf/otter.properties && sed -i 's/otter.database.driver.username = root/otter.database.driver.username = '$DB_USER'/' conf/otter.properties && sed -i 's/otter.database.driver.password = hello/otter.database.driver.password = '$DB_PASSWD'/' conf/otter.properties && sed -i 's/otter.zookeeper.cluster.default = 127.0.0.1/otter.zookeeper.cluster.default = '$ZK_IP'/' conf/otter.properties && sh bin/startup.sh && while true; do echo hello world; sleep 1000;done"]
# otter-manager
docker run --name otter-manager -d -p 1800:1800 -p 1099:1099 -e IP=114.67.233.234 -e PORT=1800 -e ZK_IP=10.0.0.3 -e DB_IP=10.0.0.3:3306 -e DB_USER=root -e DB_PASSWD=123456 otter-manager:4.2.18
4. otter-node
- otter-node不能单独起来, 需要等
otter-manger起来后, 配置好分配的id后, 才可以启动 - 镜像是公网可以拉的
docker run --name otter-node -d -p 2088:2088 -p 2090:2090 -p 9090:9090 -e id=1 -e address=10.0.0.3 aeert/otter-node:4.2.18
(二). 源数据库
-
k8s部署的跳过
-
如果是docker部署的mysql, 镜像为
mysql:5.7, 里面是没有vi命令的,需要需改
cp /etc/apt/sources.list /etc/apt/sources.list.bak
echo "deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse" > /etc/apt/sources.list
apt update
# 报错,最后面有一个子串
W: GPG error: http://mirrors.aliyun.com/ubuntu bionic InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 3B4FE6ACC0B21F32
# 替换最后的子串为上面报错的, 然后再次 apt update
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
# 安装vim
apt-get install vim
# 最后docker restart mysql-xxxx
1. mysql设置
vim /etc/mysql/conf.d/mysql.cnf
# 源mysql需要设置开启binlog,需要设置binlog_format=ROW,下面的编码根据实际情况
[mysqld]
log-bin=/var/lib/mysql/mysql-bin
binlog_format=ROW
server-id=1
character-set-server = utf8
collation-server = utf8_general_ci
- 源mysql开启binlog
SHOW VARIABLES LIKE 'log_%';
- mysql复制模式是否是row模式
SHOW VARIABLES LIKE 'binlog_format%';
- 编码
show variables like 'character_set%';
二. 配置
(一). 页面配置
- otter-manger启动好后, 浏览器访问并登陆, 默认账号密码都是: admin, 然后开始配置
- 配置zk, 点击添加
- 集群名字自定义, 不要重复, 不要中文
- node配置, 点击添加
- 机器名称自定义, 不要重复, 不要中文
- 机器端口, 设置为你启动node的端口, node默认端口为 2088
- 如果是外部IP的需要设置好, 保证网络正常
- 其他的可选择性配置
- 最后启动node, 如上面启动命令
这个编码就是node的 id
数据源配置, 源数据库, 注意目标数据库都要在这添加, 点击添加源数据库和目标数据库
- 数据源名字, 自定义, 不要重复, 不要中文
- 认证信息
- URL 按照上面提示填写
- 编码, 源数据库和目标数据库的编码必须保持一致, 不同编码同步不过去
canal配置, 不需要单独部署canal, node中已经集成了canal, 我们这只需要配置好即可
- canal名称自定义, 不重复, 不中文
- 其他按照提示填写
- 同步信息填写, 源数据库, 目标数据库都要填写
- schema name 库名, 可填: .* 表示所有库
- table name 表名, 可填: .* 表示所有表
- 选择数据源,就是我们上面添加的2个数据库名称
- 保存后, 我们需要再创建一个数据表, 为目标数据表, 表示要同步到那个数据库中
- 同步管理, 创建Channel
- Channel名称不重复, 不中文
- 其他默认保存
- 点击创建好的Channel, 添加pipeline
- pipeline名称
- 选择canal
- 其他默认即可, 保存
- 点击创建好的pipeline
- 设置表的映射关系, 如果源数据库和目标数据库同步的表名称一样, 直接保存即可
- 开始同步
- 点击启用即开始同步
(二). 全库同步配置
- otter 默认只是同步增量的数据, 如果要同步全量的数据需要做如下操作
# 在源库中 创建database retl, 供 otter 使用， otter 需要对 retl.* 的读写权限，以及对业务表的读写权限
CREATE DATABASE retl;
# 1.创建用户,及授权
CREATE USER retl@'%' IDENTIFIED BY 'retl';
GRANT USAGE ON *.* TO `retl`@'%';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO `retl`@'%';
GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON `retl`.* TO `retl`@'%';
# 2. 用户授权 给同步用户授权
# 业务表授权，这里可以限定只授权同步业务的表
GRANT SELECT, INSERT, UPDATE, DELETE ON *.* TO `retl`@'%';
# 3. 创建系统表
USE retl;
DROP TABLE IF EXISTS retl.retl_buffer;
DROP TABLE IF EXISTS retl.retl_mark;
DROP TABLE IF EXISTS retl.xdual;
# 这里如果创建失败, 在mysql的安装目录下，打开my.ini或my.cnf文件，新增 sql_mode = ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION，
CREATE TABLE retl_buffer
(
ID BIGINT(20) AUTO_INCREMENT,
TABLE_ID INT(11) NOT NULL,
FULL_NAME varchar(512),
TYPE CHAR(1) NOT NULL,
PK_DATA VARCHAR(256) NOT NULL,
GMT_CREATE TIMESTAMP NOT NULL,
GMT_MODIFIED TIMESTAMP NOT NULL,
CONSTRAINT RETL_BUFFER_ID PRIMARY KEY (ID)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE retl_mark
(
ID BIGINT AUTO_INCREMENT,
CHANNEL_ID INT(11),
CHANNEL_INFO varchar(128),
CONSTRAINT RETL_MARK_ID PRIMARY KEY (ID)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE xdual (
ID BIGINT(20) NOT NULL AUTO_INCREMENT,
X timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (ID)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
# 4. 插入初始化数据
INSERT INTO retl.xdual(id, x) VALUES (1,now()) ON DUPLICATE KEY UPDATE x = now();
# 之后的操作其实就是，把你想要同步表的表名及记录ID插入到retl_buffer表即可
# 需要注意, 插入语句必须是在同步运行中配置才会生效
insert into retl.retl_buffer(ID,TABLE_ID, FULL_NAME,TYPE,PK_DATA,GMT_CREATE,GMT_MODIFIED) (select null,0,'$schema.table$','I',id,now(),now() from $schema.table$);
1
# 如果针对多主键时，对应的PK_DATA需要将需要同步表几个主键按照(char)1进行拼接。
# 以下这条sql的意思就是将rrrr.t_test的主键为aaa的这条记录按照规则插入到目标库
insert into `retl`.`retl_buffer` ( `TABLE_ID`, `FULL_NAME`, `TYPE`, `PK_DATA`, `GMT_CREATE`, `GMT_MODIFIED`) values ( '0', 'rrrr.t_test', 'I', 'aaa', now(), now());
# 案例, 同步test库中的moon表全部信息,再次强调必须是在运行中的时候插入这条语句
# num字段为主键, I表示insert
insert into retl.retl_buffer(ID,TABLE_ID, FULL_NAME,TYPE,PK_DATA,GMT_CREATE,GMT_MODIFIED) (select null,0,'test.moon','I',num,now(),now() from test.moon);
- 源库创建
create database if not exists test default character set = 'utf8';
USE test;
CREATE TABLE `example` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
`name` VARCHAR(32) COLLATE utf8_bin DEFAULT NULL ,
PRIMARY KEY (`ID`)
) ENGINE=INNODB DEFAULT CHARSET=utf8;
insert into test.example(id,name) values(null,'hello111');