阿里开源数据同步工具--DataX
下载 地址：QuickStart
DataX是异构数据源离线同步工具。能够将MySQL sqlServer Oracle Hive HBase FTP 之间进行稳定高效的数据同步。
设计思路：
网状连接-》 星型连接
目前支持哪些数据同步？：
核心架构：
推荐使用python 2.6 7 不要使用python 3,0
使用方法和案例：
1. 准备一个job.json 或者job.xml
执行命令：
python datax.py job.json
提供几个json :
1: oracle 到本地文件
{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxxxx",
"password": "xxxxxx",
"where": "",
"connection": [
{
"querySql": [
"select * from table1"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "D:/data/",
"fileName": "oracledata",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
],
"setting": {
"speed": {
"channel": 1
}
}
}
}
2.oracle ->oracle
{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxxx",
"password": "xxxx",
"column": ["*"],
"where": "",
"connection": [
{
"querySql": [
"select * from table1"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
"name": "oraclewriter",
"parameter": {
"username": "xxxx",
"password": "xxxx",
"truncate": "true",
"column": ["SYSTEMNO",
"SYSTEMTIME",
"xxx",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"x",
"STATUS",
"x",
"x",
"x"
],
"connection": [
{
"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcexxnter",
"table": [
"table1"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 6
}
}
}
}
关联查询入库demo:
建表SQL：
CREATE TABLE ZF_PaytDemo(
SystemNo VARCHAR2(32) NOT NULL,
MainOrderSystemNo VARCHAR2(32) NOT NULL,
TxAmount NUMBER(16) NOT NULL,
AccountAmount NUMBER(16) NOT NULL,
FinanceChannelCode VARCHAR2(64) NOT NULL
);
Json:
{
"job": {
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "xxx",
"password": "xxx",
"where": "",
"connection": [
{
"querySql": [
"select B.SYSTEMNO, A.SYSTEMNO AS MainOrderSystemNo, B.txamount, B.AccountAmount, B.Financechannelcode from zf_fundinordermain A left join zf_fundinorderpayt B on A.Systemno = B.MAINORDERSYSTEMNO where A.Systemno ='35112016091411580603700000' "
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
]
}
]
}
},
"writer": {
"name": "oraclewriter",
"parameter": {
"username": "xxx",
"password": "xxx",
"truncate": "false",
"column": [
"SystemNo",
"MainOrderSystemNo",
"TxAmount",
"AccountAmount",
"FinanceChannelCode"
],
"connection": [
{
"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
"table": [
"ZF_PaytDemo"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 1
}
}
}
}
file->oracle JSON:
{
"job": {
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["D:/datax.txt"],
"encoding": "UTF-8",
"column": [
{
"index": 0,
"type": "string"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "long"
},
{
"index": 3,
"type": "long"
},
{
"index": 4,
"type": "string"
}
],
"fieldDelimiter": "\t"
}
},
"writer": {
"name": "oraclewriter",
"parameter": {
"username": "xxxx",
"password": "xxxx",
"truncate": "true",
"column": [
"SystemNo",
"MainOrderSystemNo",
"TxAmount",
"AccountAmount",
"FinanceChannelCode"
],
"connection": [
{
"jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
"table": [
"ZF_PaytDemo"
]
}
]
}
}
}
],
"setting": {
"speed": {
"channel": 6
}
}
}
}
原文：https://blog.csdn.net/wangming520liwei/article/details/70159537