DataX---数据同步json操作
成功！mysql到本地打印控制台
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": false,
"encoding": "UTF-8"
}
}
}
]
}
}
任务启动时刻 : 2016-09-29 14:20:28
任务结束时刻 : 2016-09-29 14:20:52
任务总计耗时 : 23s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0
任务启动时刻 : 2016-09-29 14:31:50
任务结束时刻 : 2016-09-29 14:32:01
任务总计耗时 : 10s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0
{
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"jdbcUrl": ["jdbc:mysql://192.168.0.197:3306/qyyx_test"],
"table":[visit_log]
}
],
"where":"select ID,VISIT_IP from visit_log where ID < 9200"
}
成功！本地文件到本地文件
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/home/haiwei.luo/case00/data"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": ","
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/home/haiwei.luo/case00/result",
"fileName": "luohw",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
]
}
}
成功！HDFS打印到本地控制台
{
"job": {
"setting": {
"speed": {
"channel": 3
}
},
"content": [
{
"reader": {
"name": "hdfsreader",
"parameter": {
"path": "/user/hive/warehouse/test.db/nnn/*",
"defaultFS": "hdfs://192.168.200.45:8020",
"column": ["*"],
"fileType": "orc",
"encoding": "UTF-8",
"fieldDelimiter": ","
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": true
}
}
}
]
}
}
成功！oracle到本地文件
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP","BEF_URL","NOW_URL","CORP_CODE","PROMOTE_NO","PLATFORM","COOKIE_EANBLED","USER_AGENT","APP_VERSION","APP_NAME","COLOR","SCREEN","JAVA_ENABLED","JS_VERSION","SID","CREATE_BY","CREATE_TIME","MODIFY_BY","MODIFY_TIME"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
],
"setting": {
"speed": {
"channel": 10
}
}
}
}
失败！本地文件到HDFS
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/opt/result"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": "\t"
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}
}
]
}
}
成功：orcle打印到Linux
{
"job": {
"setting": {
"speed": {
"channel": 5
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"visible": true,
"encoding": "UTF-8"
}
}
}
]
}
}
oracle到HDFS
失败！待修改
{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}
}
]
}
}
成功！oracle到本地文件
{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
]
}
}
channel：5
数据量：8906745556=8.3GB
2016-09-29 17:25:39.146 [job-0] INFO JobContainer - PerfTrace not enable!
2016-09-29 17:25:39.146 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 2.157s | All Task WaitReaderTime 758.762s | Percentage 100.00%
2016-09-29 17:25:39.147 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:09:58
任务结束时刻 : 2016-09-29 17:25:39
任务总计耗时 : 940s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0
channel：200
数据量：8906745556=8.3GB
2016-09-29 17:44:14.648 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 1.418s | All Task WaitReaderTime 759.494s | Percentage 100.00%
2016-09-29 17:44:14.649 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:28:33
任务结束时刻 : 2016-09-29 17:44:14
任务总计耗时 : 941s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": false,
"encoding": "UTF-8"
}
}
}
]
}
}
任务启动时刻 : 2016-09-29 14:20:28
任务结束时刻 : 2016-09-29 14:20:52
任务总计耗时 : 23s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0
任务启动时刻 : 2016-09-29 14:31:50
任务结束时刻 : 2016-09-29 14:32:01
任务总计耗时 : 10s
任务平均流量 : 364B/s
记录写入速度 : 3rec/s
读出记录总数 : 31
读写失败总数 : 0
{
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"connection": [
{
"jdbcUrl": ["jdbc:mysql://192.168.0.197:3306/qyyx_test"],
"table":[visit_log]
}
],
"where":"select ID,VISIT_IP from visit_log where ID < 9200"
}
成功！本地文件到本地文件
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/home/haiwei.luo/case00/data"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": ","
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/home/haiwei.luo/case00/result",
"fileName": "luohw",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
]
}
}
成功！HDFS打印到本地控制台
{
"job": {
"setting": {
"speed": {
"channel": 3
}
},
"content": [
{
"reader": {
"name": "hdfsreader",
"parameter": {
"path": "/user/hive/warehouse/test.db/nnn/*",
"defaultFS": "hdfs://192.168.200.45:8020",
"column": ["*"],
"fileType": "orc",
"encoding": "UTF-8",
"fieldDelimiter": ","
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"print": true
}
}
}
]
}
}
成功！oracle到本地文件
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "root",
"password": "root",
"column": ["ID","VISIT_IP","BEF_URL","NOW_URL","CORP_CODE","PROMOTE_NO","PLATFORM","COOKIE_EANBLED","USER_AGENT","APP_VERSION","APP_NAME","COLOR","SCREEN","JAVA_ENABLED","JS_VERSION","SID","CREATE_BY","CREATE_TIME","MODIFY_BY","MODIFY_TIME"],
"connection": [
{
"table": [
"visit_log"
],
"jdbcUrl": [
"jdbc:mysql://192.168.0.197:3306/qyyx_test"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
],
"setting": {
"speed": {
"channel": 10
}
}
}
}
失败！本地文件到HDFS
{
"setting": {},
"job": {
"setting": {
"speed": {
"channel": 2
}
},
"content": [
{
"reader": {
"name": "txtfilereader",
"parameter": {
"path": ["/opt/result"],
"encoding": "UTF-8",
"column": ["*"],
"fieldDelimiter": "\t"
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}
}
]
}
}
成功：orcle打印到Linux
{
"job": {
"setting": {
"speed": {
"channel": 5
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "streamwriter",
"parameter": {
"visible": true,
"encoding": "UTF-8"
}
}
}
]
}
}
oracle到HDFS
失败！待修改
{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://192.168.200.45:8020",
"fileType": "text",
"path": "/user/hive/warehouse/test.db/nnn/",
"fileName": "text",
"column": ["*"],
"writeMode": "append",
"fieldDelimiter": "\t",
"compress":"NONE"
}
}
}
]
}
}
成功！oracle到本地文件
{
"job": {
"setting": {
"speed": {
"channel": 200
}
},
"content": [
{
"reader": {
"name": "oraclereader",
"parameter": {
"username": "edw_src",
"password": "123456",
"where": "",
"connection": [
{
"querySql": [
"select * from ORDER_DETAIL_BAK"
],
"jdbcUrl": [
"jdbc:oracle:thin:@192.168.0.197:1521:orcl"
]
}
]
}
},
"writer": {
"name": "txtfilewriter",
"parameter": {
"path": "/opt/result/",
"fileName": "shun",
"writeMode": "truncate",
"format": "yyyy-MM-dd"
}
}
}
]
}
}
channel：5
数据量：8906745556=8.3GB
2016-09-29 17:25:39.146 [job-0] INFO JobContainer - PerfTrace not enable!
2016-09-29 17:25:39.146 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 2.157s | All Task WaitReaderTime 758.762s | Percentage 100.00%
2016-09-29 17:25:39.147 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:09:58
任务结束时刻 : 2016-09-29 17:25:39
任务总计耗时 : 940s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0
channel：200
数据量：8906745556=8.3GB
2016-09-29 17:44:14.648 [job-0] INFO StandAloneJobContainerCommunicator - Total 20238811 records, 4580007717 bytes | Speed 4.65MB/s, 21530 records/s | Error 0 records, 0 bytes | All Task WaitWriterTime 1.418s | All Task WaitReaderTime 759.494s | Percentage 100.00%
2016-09-29 17:44:14.649 [job-0] INFO JobContainer -
任务启动时刻 : 2016-09-29 17:28:33
任务结束时刻 : 2016-09-29 17:44:14
任务总计耗时 : 941s
任务平均流量 : 4.65MB/s
记录写入速度 : 21530rec/s
读出记录总数 : 20238811
读写失败总数 : 0