oracle GBK字符迁移lightdb-x测试
E:\HS\LightDBSVN\23.3sql文件\迁移工具\caofa\config\application.properties
gbk-->uft8:
logging.config=classpath:log4j2.xml
# ????
etl.global.sourceDatabase=oracle
etl.global.targetDatabase=lightdb
etl.global.showSql=false
etl.global.fastFail=false
etl.global.enableWithDoubleQuote=false
etl.global.package.onlyKeepType=true
etl.global.transferSchemas=hr
etl.global.enableFunctions=TransferTable,TransferTableData,TableCompare,TableDataCompare,TransferIndex,TransferSequence,TransferView,TransferTrigger,TransferPackage,TransferProcedure,TransferFunction
#gbk
etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/gbk
etl.source.jdbc.username=test
etl.source.jdbc.password=test
# utf8
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5432/etlu2
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123
-- 创建表
drop TABLE hr.gbk2;
CREATE TABLE hr.gbk2 (
id NUMBER PRIMARY KEY,
data VARCHAR2(4000)
);
INSERT INTO hr.gbk2 (id, data) VALUES (1, 'fhsjdhfjs看到回國的飛機共和黨國會');
INSERT INTO hr.gbk2 (id, data) VALUES (2, '付款的官方的理工科 ');
INSERT INTO hr.gbk2 (id, data) VALUES (3, '看來是價格 ');
INSERT INTO hr.gbk2 (id, data) VALUES (4, '快攻打法瞭解更多 ');
INSERT INTO hr.gbk2 (id, data) VALUES (5, '84375jkdgkfckvkb堅不可摧v ');
INSERT INTO hr.gbk2 (id, data) VALUES (6, '；報價參考報價下 ');
INSERT INTO hr.gbk2 (id, data) VALUES (7, '老闆超級不開心 ');
INSERT INTO hr.gbk2 (id, data) VALUES (8, '了從包括v菠蘿快車v了 ');
INSERT INTO hr.gbk2 (id, data) VALUES (9, '从离开滨江屡见不鲜 ');
INSERT INTO hr.gbk2 (id, data) VALUES (10, '查看背景下来不及 差旅不可撤销吧');
INSERT INTO hr.gbk2 (id, data) VALUES (11, 'clvjbxj ');
INSERT INTO hr.gbk2 (id, data) VALUES (12, '了出现崩溃');
INSERT INTO hr.gbk2 (id, data) VALUES (13, '查看下礼拜');
commit;
select * from hr.gbk2;插入自定义字符：
# 通过unicode插入不存在gbk unicode_char4 = u'\ue000'
insert_sql = """
INSERT INTO hr.gbk2 (id, data)
VALUES (:id1,:value)
"""
cursor.execute(insert_sql, {'id1':37,'value': unicode_char4})
迁移后查看lightdb-x：
lightdb@etlu2=# drop table hr.gbk2;
DROP TABLE
lightdb@etlu2=# select * from hr.gbk2;
ERROR: relation "hr.gbk2" does not exist
LINE 1: select * from hr.gbk2;
^
lightdb@etlu2=# select * from hr.gbk2;
id | data
----+-----------------------------------
37 |
1 | fhsjdhfjs看到回國的飛機共和黨國會
2 | 付款的官方的理工科
3 | 看來是價格
4 | 快攻打法瞭解更多
5 | 84375jkdgkfckvkb堅不可摧v
6 | ；報價參考報價下
7 | 老闆超級不開心
8 | 了從包括v菠蘿快車v了
9 | 从离开滨江屡见不鲜
10 | 查看背景下来不及 差旅不可撤销吧
11 | clvjbxj
12 | 了出现崩溃
13 | 查看下礼拜
36 |
(15 rows)
lightdb@etlu2=# show server_encoding ;
server_encoding
-----------------
UTF8
(1 row)
lightdb@etlu2=#
2023-09-27 17:50:38.276 [main] INFO com.hundsun.lightdb.core.EtlDirector - 表结构对比开始
2023-09-27 17:50:38.277 [main] INFO com.hundsun.lightdb.core.EtlDirector - hr.countries对比一致。hr.regions对比一致。hr.gbk2对比一致。hr.gbktest对比一致。
数据比对一致：
2023-09-27 17:50:38.795 [main] INFO com.hundsun.lightdb.core.EtlDirector - 数据对比开始
regions对比一致。countries对比一致。job_history对比一致。locations对比一致。gbk2对比一致。jobs对比一致。employees对比一致。departments对比一致。
gbk--->gbk:
[lightdb@lightdb1 lightdb-x-13.8-23.3-14917-el7.x86_64]$ /data2/xjh/auto/lightdb-x/13.8-23.3/bin/ltsql -p 5444
ltsql (13.8-23.3)
compatible type: postgresql
Type "help" for help.
lightdb@postgres=# \l
List of databases
Name | Owner | Encoding | Collate | Ctype | Access privileges
-----------+---------+----------+-----------+-----------+---------------------
etl | lightdb | GBK | zh_CN.gbk | zh_CN.gbk | =Tc/lightdb +
| | | | | lightdb=CTc/lightdb+
| | | | | etl=CTc/lightdb
etl_gbk | lightdb | GBK | zh_CN.gbk | zh_CN.gbk | =Tc/lightdb +
| | | | | lightdb=CTc/lightdb+
| | | | | etl_gbk=CTc/lightdb
lt_test | lightdb | GBK | zh_CN.gbk | zh_CN.gbk |
postgres | lightdb | GBK | zh_CN.gbk | zh_CN.gbk |
template0 | lightdb | GBK | zh_CN.gbk | zh_CN.gbk | =c/lightdb +
| | | | | lightdb=CTc/lightdb
template1 | lightdb | GBK | zh_CN.gbk | zh_CN.gbk | =c/lightdb +
| | | | | lightdb=CTc/lightdb
(6 rows)
lightdb@postgres=# \c etl_gbk;
You are now connected to database "etl_gbk" as user "lightdb".
compatible type: oracle
lightdb@etl_gbk=# show server_encoding ;
server_encoding
-----------------
GBK
(1 row)
lightdb@etl_gbk=#
修改迁移工具配置：
# gbk
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5444/etl_gbk
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123发生中断：
2023-09-27 18:02:03.066 [pool-6-thread-1] INFO com.hundsun.lightdb.JdbcUtils - 执行SQL CREATE TABLE HR.COUNTRIES (COUNTRY_ID char(2) NOT NULL,COUNTRY_NAME varchar2(40),REGION_ID number,CONSTRAINT COUNTRY_C_ID_PK PRIMARY KEY(COUNTRY_ID))
2023-09-27 18:02:03.073 [pool-6-thread-1] WARN com.hundsun.lightdb.core.EtlGlobalCtx - 关系已存在
堆栈：org.postgresql.util.PSQLException: ERROR: relation "countries" already exists
at org.postgresql.core.v3.QueryExecutorImpl.ALLATORIxDEMO(le:3279)
at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:1889)
at org.postgresql.core.v3.QueryExecutorImpl.execute(le:1076)
at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:1454)
at org.postgresql.jdbc.PgStatement.execute(tl:866)
at org.postgresql.jdbc.PgStatement.executeWithFlags(tl:1201)
at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:521)
at org.postgresql.jdbc.PgStatement.executeWithFlags(tl:881)
at org.postgresql.jdbc.PgStatement.execute(tl:749)
at com.hundsun.lightdb.JdbcUtils.executeSql(JdbcUtils.java:231)
at com.hundsun.lightdb.core.lightdb.LightdbTargetJdbcMeta.executeSql(LightdbTargetJdbcMeta.java:78)
at com.hundsun.lightdb.core.lightdb.LightdbTargetImporter.lambda$execute$0(LightdbTargetImporter.java:47)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
2023-09-27 18:02:03.174 [main] ERROR com.hundsun.lightdb.core.EtlGlobalCtx - 信息: 任务执行中断
堆栈：java.lang.NullPointerException
at com.hundsun.lightdb.JdbcUtils.fetchTargetInfo(JdbcUtils.java:179)
at com.hundsun.lightdb.core.lightdb.LightdbTargetJdbcMeta.columnInfo(LightdbTargetJdbcMeta.java:129)
at com.hundsun.lightdb.core.lightdb.LightdbTargetImporter.columnInfo(LightdbTargetImporter.java:65)
at com.hundsun.lightdb.core.EtlDirector.begin(EtlDirector.java:56)
at com.hundsun.lightdb.core.EtlDirector.run(EtlDirector.java:419)
at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:775)
at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:765)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:322)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1226)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1215)
at com.hundsun.lightdb.ProcedureTransformerApplication.main(ProcedureTransformerApplication.java:15)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:108)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:65)
2023-09-27 18:02:03.190 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection oracle.jdbc.driver.T4CConnection@4905c46b
2023-09-27 18:02:03.190 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@58496dc
2023-09-27 18:02:03.191 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@151db587
2023-09-27 18:02:03.191 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@238acd0b
2023-09-27 18:02:03.192 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@23811a09
2023-09-27 18:02:03.192 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@2125ad3
2023-09-27 18:02:03.194 [main] INFO com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@7a5b769b
2023-09-27 18:02:03.196 [SpringContextShutdownHook] INFO org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor - Shutting down ExecutorService 'applicationTaskExecutor'
2023-09-27 18:02:03.229 [Thread-3] WARN org.ehcache.impl.persistence.DefaultLocalPersistenceService - Could not delete all file based persistence contexts owned by file - is not a directory!
2023-09-27 18:02:03.233 [Thread-3] INFO org.ehcache.core.EhcacheManager - All persistent data destroyed for EhcacheManageruft8-->gbk:报错
#utf-8
etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/test
etl.source.jdbc.username=hr
etl.source.jdbc.password=hr
# gbk
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5444/etl_gbk
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123表结构迁移成功，数据迁移失败。
2023-09-27 18:11:15.664 [pool-7-thread-1] INFO com.hundsun.lightdb.ImportWorker - end transfer table [hr.xjh_example_table3 耗时 0 秒]
2023-09-27 18:11:15.664 [pool-4-thread-1] ERROR com.hundsun.lightdb.core.EtlGlobalCtx - 信息: error data transmission for table gbk2
堆栈：java.sql.BatchUpdateException: Batch entry 0 insert into hr.gbk2 (id,data) values ('36'::numeric,'') was aborted: ERROR: character with byte sequence 0xee 0x80 0x80 in encoding "UTF8" has no equivalent in encoding "GBK" Call getNextException to see other errors in the batch.
at org.postgresql.jdbc.BatchResultHandler.handleError(zj:182)
at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:3125)
at org.postgresql.core.v3.QueryExecutorImpl.execute(le:659)
at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:1763)
at org.postgresql.jdbc.PgStatement.executeBatch(tl:1641)
at org.postgresql.jdbc.PgPreparedStatement.executeBatch(fj:1540)
at com.hundsun.lightdb.ImportWorker.lambda$run$0(ImportWorker.java:75)
at com.hundsun.lightdb.JdbcUtils.doQuery(JdbcUtils.java:31)
at com.hundsun.lightdb.ImportWorker.run(ImportWorker.java:43)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.postgresql.util.PSQLException: ERROR: character with byte sequence 0xee 0x80 0x80 in encoding "UTF8" has no equivalent in encoding "GBK"
at org.postgresql.core.v3.QueryExecutorImpl.ALLATORIxDEMO(le:3279)
at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:1889)
... 12 more
2023-09-27 18:11:15.664 [pool-7-thread-1] INFO com.hundsun.lightdb.ImportWorker - begin data transmission for table [ hr.employee_data ]
2023-09-27 18:11:15.666 [pool-4-thread-1] INFO com.hundsun.lightdb.ImportWorker - end transfer table [hr.gbk2 耗时 0 秒]uft8迁移到utf8:
#utf-8
etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/test
etl.source.jdbc.username=hr
etl.source.jdbc.password=hr
# utf8
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5432/etlu2
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123
[lightdb@lightdb1 lightdb-x-13.8-23.3-14917-el7.x86_64]$ /data2/xjh/auto/lightdb-x/13.8-23.3/bin/ltsql -p 5432
ltsql (13.8-23.3)
compatible type: postgresql
Type "help" for help.
lightdb@postgres=# \c etlu2
You are now connected to database "etlu2" as user "lightdb".
compatible type: oracle
lightdb@etlu2=# show port;
port
------
5432
(1 row)
lightdb@etlu2=# show server_encoding ;
server_encoding
-----------------
UTF8
(1 row)
lightdb@etlu2=# select * from hr.gbk2;
id | data
----+-----------------------------------
37 |
1 | fhsjdhfjs看到回國的飛機共和黨國會
2 | 付款的官方的理工科
3 | 看來是價格
4 | 快攻打法瞭解更多
5 | 84375jkdgkfckvkb堅不可摧v
6 | ；報價參考報價下
7 | 老闆超級不開心
8 | 了從包括v菠蘿快車v了
9 | 从离开滨江屡见不鲜
10 | 查看背景下来不及 差旅不可撤销吧
11 | clvjbxj
12 | 了出现崩溃
13 | 查看下礼拜
36 |
(15 rows)
lightdb@etlu2=# drop table hr.gbk2;
DROP TABLE
lightdb@etlu2=#
迁移数据成功：
lightdb@etlu2=# select * from hr.gbk2;
id | data
----+-----------------------------------
36 |
37 |
1 | fhsjdhfjs看到回國的飛機共和黨國會
2 | 付款的官方的理工科
3 | 看來是價格
4 | 快攻打法瞭解更多
5 | 84375jkdgkfckvkb堅不可摧v
6 | ；報價參考報價下
7 | 老闆超級不開心
8 | 了從包括v菠蘿快車v了
9 | 从离开滨江屡见不鲜
10 | 查看背景下来不及 差旅不可撤销吧
11 | clvjbxj
12 | 了出现崩溃
13 | 查看下礼拜
(15 rows)
lightdb@etlu2=#