oracle GBK字符迁移lightdb-x测试

E:\HS\LightDBSVN\23.3sql文件\迁移工具\caofa\config\application.properties

gbk-->uft8:

logging.config=classpath:log4j2.xml
# ????
etl.global.sourceDatabase=oracle
etl.global.targetDatabase=lightdb
etl.global.showSql=false
etl.global.fastFail=false
etl.global.enableWithDoubleQuote=false
etl.global.package.onlyKeepType=true
etl.global.transferSchemas=hr
etl.global.enableFunctions=TransferTable,TransferTableData,TableCompare,TableDataCompare,TransferIndex,TransferSequence,TransferView,TransferTrigger,TransferPackage,TransferProcedure,TransferFunction


#gbk

etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/gbk
etl.source.jdbc.username=test
etl.source.jdbc.password=test

# utf8
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5432/etlu2
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123

-- 创建表  
drop TABLE hr.gbk2;
CREATE  TABLE hr.gbk2 (  
  id NUMBER PRIMARY KEY,  
  data VARCHAR2(4000)  
);  

INSERT INTO hr.gbk2 (id, data) VALUES (1, 'fhsjdhfjs看到回國的飛機共和黨國會');
INSERT INTO hr.gbk2 (id, data) VALUES (2, '付款的官方的理工科              ');
INSERT INTO hr.gbk2 (id, data) VALUES (3, '看來是價格                      ');
INSERT INTO hr.gbk2 (id, data) VALUES (4, '快攻打法瞭解更多                ');
INSERT INTO hr.gbk2 (id, data) VALUES (5, '84375jkdgkfckvkb堅不可摧v       ');
INSERT INTO hr.gbk2 (id, data) VALUES (6, '；報價參考報價下                ');
INSERT INTO hr.gbk2 (id, data) VALUES (7, '老闆超級不開心                  ');
INSERT INTO hr.gbk2 (id, data) VALUES (8, '了從包括v菠蘿快車v了            ');
INSERT INTO hr.gbk2 (id, data) VALUES (9, '从离开滨江屡见不鲜              ');
INSERT INTO hr.gbk2 (id, data) VALUES (10, '查看背景下来不及  差旅不可撤销吧');
INSERT INTO hr.gbk2 (id, data) VALUES (11, 'clvjbxj   ');
INSERT INTO hr.gbk2 (id, data) VALUES (12, '了出现崩溃');
INSERT INTO hr.gbk2 (id, data) VALUES (13, '查看下礼拜');
commit;

select * from hr.gbk2;

插入自定义字符：

# 通过unicode插入不存在gbk
unicode_char4 = u'\ue000'

insert_sql = """  
    INSERT INTO hr.gbk2 (id, data)  
    VALUES (:id1,:value)  
"""

cursor.execute(insert_sql, {'id1':37,'value': unicode_char4})

迁移后查看lightdb-x：

lightdb@etlu2=# drop table hr.gbk2;
DROP TABLE
lightdb@etlu2=# select * from hr.gbk2;
ERROR:  relation "hr.gbk2" does not exist
LINE 1: select * from hr.gbk2;
                      ^
lightdb@etlu2=# select * from hr.gbk2;
 id |               data                
----+-----------------------------------
 37 | 
  1 | fhsjdhfjs看到回國的飛機共和黨國會
  2 | 付款的官方的理工科              
  3 | 看來是價格                      
  4 | 快攻打法瞭解更多                
  5 | 84375jkdgkfckvkb堅不可摧v       
  6 | ；報價參考報價下                
  7 | 老闆超級不開心                  
  8 | 了從包括v菠蘿快車v了            
  9 | 从离开滨江屡见不鲜              
 10 | 查看背景下来不及  差旅不可撤销吧
 11 | clvjbxj   
 12 | 了出现崩溃
 13 | 查看下礼拜
 36 | 
(15 rows)

lightdb@etlu2=# show server_encoding ;
 server_encoding 
-----------------
 UTF8
(1 row)

lightdb@etlu2=# 

2023-09-27 17:50:38.276 [main] INFO  com.hundsun.lightdb.core.EtlDirector - 表结构对比开始
2023-09-27 17:50:38.277 [main] INFO  com.hundsun.lightdb.core.EtlDirector - hr.countries对比一致。hr.regions对比一致。hr.gbk2对比一致。hr.gbktest对比一致。

数据比对一致：

2023-09-27 17:50:38.795 [main] INFO  com.hundsun.lightdb.core.EtlDirector - 数据对比开始
regions对比一致。countries对比一致。job_history对比一致。locations对比一致。gbk2对比一致。jobs对比一致。employees对比一致。departments对比一致。

gbk--->gbk:

[lightdb@lightdb1 lightdb-x-13.8-23.3-14917-el7.x86_64]$ /data2/xjh/auto/lightdb-x/13.8-23.3/bin/ltsql -p 5444
ltsql (13.8-23.3)
compatible type: postgresql
Type "help" for help.

lightdb@postgres=# \l
                              List of databases
   Name    |  Owner  | Encoding |  Collate  |   Ctype   |  Access privileges  
-----------+---------+----------+-----------+-----------+---------------------
 etl       | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | =Tc/lightdb        +
           |         |          |           |           | lightdb=CTc/lightdb+
           |         |          |           |           | etl=CTc/lightdb
 etl_gbk   | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | =Tc/lightdb        +
           |         |          |           |           | lightdb=CTc/lightdb+
           |         |          |           |           | etl_gbk=CTc/lightdb
 lt_test   | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | 
 postgres  | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | 
 template0 | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | =c/lightdb         +
           |         |          |           |           | lightdb=CTc/lightdb
 template1 | lightdb | GBK      | zh_CN.gbk | zh_CN.gbk | =c/lightdb         +
           |         |          |           |           | lightdb=CTc/lightdb
(6 rows)

lightdb@postgres=# \c etl_gbk;
You are now connected to database "etl_gbk" as user "lightdb".
compatible type: oracle
lightdb@etl_gbk=# show server_encoding ;
 server_encoding 
-----------------
 GBK
(1 row)

lightdb@etl_gbk=# 

修改迁移工具配置：

# gbk
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5444/etl_gbk
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123

发生中断：

2023-09-27 18:02:03.066 [pool-6-thread-1] INFO  com.hundsun.lightdb.JdbcUtils - 执行SQL CREATE TABLE HR.COUNTRIES (COUNTRY_ID char(2) NOT NULL,COUNTRY_NAME varchar2(40),REGION_ID number,CONSTRAINT COUNTRY_C_ID_PK PRIMARY KEY(COUNTRY_ID))
2023-09-27 18:02:03.073 [pool-6-thread-1] WARN  com.hundsun.lightdb.core.EtlGlobalCtx - 关系已存在
 堆栈：org.postgresql.util.PSQLException: ERROR: relation "countries" already exists
        at org.postgresql.core.v3.QueryExecutorImpl.ALLATORIxDEMO(le:3279)
        at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:1889)
        at org.postgresql.core.v3.QueryExecutorImpl.execute(le:1076)
        at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:1454)
        at org.postgresql.jdbc.PgStatement.execute(tl:866)
        at org.postgresql.jdbc.PgStatement.executeWithFlags(tl:1201)
        at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:521)
        at org.postgresql.jdbc.PgStatement.executeWithFlags(tl:881)
        at org.postgresql.jdbc.PgStatement.execute(tl:749)
        at com.hundsun.lightdb.JdbcUtils.executeSql(JdbcUtils.java:231)
        at com.hundsun.lightdb.core.lightdb.LightdbTargetJdbcMeta.executeSql(LightdbTargetJdbcMeta.java:78)
        at com.hundsun.lightdb.core.lightdb.LightdbTargetImporter.lambda$execute$0(LightdbTargetImporter.java:47)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:834)

2023-09-27 18:02:03.174 [main] ERROR com.hundsun.lightdb.core.EtlGlobalCtx - 信息: 任务执行中断
 堆栈：java.lang.NullPointerException
        at com.hundsun.lightdb.JdbcUtils.fetchTargetInfo(JdbcUtils.java:179)
        at com.hundsun.lightdb.core.lightdb.LightdbTargetJdbcMeta.columnInfo(LightdbTargetJdbcMeta.java:129)
        at com.hundsun.lightdb.core.lightdb.LightdbTargetImporter.columnInfo(LightdbTargetImporter.java:65)
        at com.hundsun.lightdb.core.EtlDirector.begin(EtlDirector.java:56)
        at com.hundsun.lightdb.core.EtlDirector.run(EtlDirector.java:419)
        at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:775)
        at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:765)
        at org.springframework.boot.SpringApplication.run(SpringApplication.java:322)
        at org.springframework.boot.SpringApplication.run(SpringApplication.java:1226)
        at org.springframework.boot.SpringApplication.run(SpringApplication.java:1215)
        at com.hundsun.lightdb.ProcedureTransformerApplication.main(ProcedureTransformerApplication.java:15)
        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
        at org.springframework.boot.loader.Launcher.launch(Launcher.java:108)
        at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
        at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:65)

2023-09-27 18:02:03.190 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection oracle.jdbc.driver.T4CConnection@4905c46b
2023-09-27 18:02:03.190 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@58496dc
2023-09-27 18:02:03.191 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@151db587
2023-09-27 18:02:03.191 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@238acd0b
2023-09-27 18:02:03.192 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@23811a09
2023-09-27 18:02:03.192 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@2125ad3
2023-09-27 18:02:03.194 [main] INFO  com.hundsun.lightdb.core.EtlGlobalCtx - close connection org.postgresql.jdbc.PgConnection@7a5b769b
2023-09-27 18:02:03.196 [SpringContextShutdownHook] INFO  org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor - Shutting down ExecutorService 'applicationTaskExecutor'
2023-09-27 18:02:03.229 [Thread-3] WARN  org.ehcache.impl.persistence.DefaultLocalPersistenceService - Could not delete all file based persistence contexts owned by file - is not a directory!
2023-09-27 18:02:03.233 [Thread-3] INFO  org.ehcache.core.EhcacheManager - All persistent data destroyed for EhcacheManager

uft8-->gbk:报错

#utf-8
etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/test
etl.source.jdbc.username=hr
etl.source.jdbc.password=hr


# gbk
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5444/etl_gbk
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123

表结构迁移成功，数据迁移失败。

2023-09-27 18:11:15.664 [pool-7-thread-1] INFO  com.hundsun.lightdb.ImportWorker - end transfer table [hr.xjh_example_table3 耗时 0 秒]
2023-09-27 18:11:15.664 [pool-4-thread-1] ERROR com.hundsun.lightdb.core.EtlGlobalCtx - 信息: error data transmission for table gbk2 
 堆栈：java.sql.BatchUpdateException: Batch entry 0 insert into hr.gbk2 (id,data) values ('36'::numeric,'') was aborted: ERROR: character with byte sequence 0xee 0x80 0x80 in encoding "UTF8" has no equivalent in encoding "GBK"  Call getNextException to see other errors in the batch.
	at org.postgresql.jdbc.BatchResultHandler.handleError(zj:182)
	at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:3125)
	at org.postgresql.core.v3.QueryExecutorImpl.execute(le:659)
	at org.postgresql.jdbc.PgStatement.ALLATORIxDEMO(tl:1763)
	at org.postgresql.jdbc.PgStatement.executeBatch(tl:1641)
	at org.postgresql.jdbc.PgPreparedStatement.executeBatch(fj:1540)
	at com.hundsun.lightdb.ImportWorker.lambda$run$0(ImportWorker.java:75)
	at com.hundsun.lightdb.JdbcUtils.doQuery(JdbcUtils.java:31)
	at com.hundsun.lightdb.ImportWorker.run(ImportWorker.java:43)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.postgresql.util.PSQLException: ERROR: character with byte sequence 0xee 0x80 0x80 in encoding "UTF8" has no equivalent in encoding "GBK"
	at org.postgresql.core.v3.QueryExecutorImpl.ALLATORIxDEMO(le:3279)
	at org.postgresql.core.v3.QueryExecutorImpl.processResults(le:1889)
	... 12 more

2023-09-27 18:11:15.664 [pool-7-thread-1] INFO  com.hundsun.lightdb.ImportWorker - begin data transmission for table [ hr.employee_data ]
2023-09-27 18:11:15.666 [pool-4-thread-1] INFO  com.hundsun.lightdb.ImportWorker - end transfer table [hr.gbk2 耗时 0 秒]

uft8迁移到utf8:

#utf-8
etl.source.jdbc.url=jdbc:oracle:thin:@10.20.30.199:1521/test
etl.source.jdbc.username=hr
etl.source.jdbc.password=hr


# utf8
etl.target.jdbc.url=jdbc:postgresql://10.19.69.252:5432/etlu2
etl.target.jdbc.username=lightdb
etl.target.jdbc.password=lightdb123

[lightdb@lightdb1 lightdb-x-13.8-23.3-14917-el7.x86_64]$ /data2/xjh/auto/lightdb-x/13.8-23.3/bin/ltsql -p 5432
ltsql (13.8-23.3)
compatible type: postgresql
Type "help" for help.

lightdb@postgres=# \c etlu2
You are now connected to database "etlu2" as user "lightdb".
compatible type: oracle
lightdb@etlu2=# show port;
 port 
------
 5432
(1 row)

lightdb@etlu2=# show server_encoding ;
 server_encoding 
-----------------
 UTF8
(1 row)

lightdb@etlu2=# select * from hr.gbk2;
 id |               data                
----+-----------------------------------
 37 | 
  1 | fhsjdhfjs看到回國的飛機共和黨國會
  2 | 付款的官方的理工科              
  3 | 看來是價格                      
  4 | 快攻打法瞭解更多                
  5 | 84375jkdgkfckvkb堅不可摧v       
  6 | ；報價參考報價下                
  7 | 老闆超級不開心                  
  8 | 了從包括v菠蘿快車v了            
  9 | 从离开滨江屡见不鲜              
 10 | 查看背景下来不及  差旅不可撤销吧
 11 | clvjbxj   
 12 | 了出现崩溃
 13 | 查看下礼拜
 36 | 
(15 rows)

lightdb@etlu2=# drop table hr.gbk2;
DROP TABLE
lightdb@etlu2=# 

迁移数据成功：

lightdb@etlu2=# select * from hr.gbk2;
 id |               data                
----+-----------------------------------
 36 | 
 37 | 
  1 | fhsjdhfjs看到回國的飛機共和黨國會
  2 | 付款的官方的理工科              
  3 | 看來是價格                      
  4 | 快攻打法瞭解更多                
  5 | 84375jkdgkfckvkb堅不可摧v       
  6 | ；報價參考報價下                
  7 | 老闆超級不開心                  
  8 | 了從包括v菠蘿快車v了            
  9 | 从离开滨江屡见不鲜              
 10 | 查看背景下来不及  差旅不可撤销吧
 11 | clvjbxj   
 12 | 了出现崩溃
 13 | 查看下礼拜
(15 rows)

lightdb@etlu2=#