将本地mysql数据库表格导入到odps大数据计算服务项目中

将本地mysql数据库表格导入到odps大数据计算服务项目中

一、工具下载以及部署

• 方法一、直接下载DataX工具包(如果仅是使用，推荐直接下载)：

  1.1. 下载DataX :

    wget http://datax-opensource.oss-cn-hangzhou.aliyuncs.com/datax.tar.gz

  1.2. 解压 DataX：

    tar zxvf datax.tar.gz

  1.3. 修改权限为755：

    chmod -R 755 datax

二、配置示例：从MySQL读取数据 写入ODPS

 cd datax/job

  vim job.json

 

• 第一步、创建作业的配置文件（json格式）

  {
      "job": {
          "content": [
              {
                  "reader": {
                      "name": "mysqlreader",
                      "parameter": {
                          "column": [],
                          "connection": [
                              {
                                  "jdbcUrl": [],
                                  "table": []
                              }
                          ],
                          "password": "",
                          "username": "",
                          "where": ""
                      }
                  },
                  "writer": {
                      "name": "odpswriter",
                      "parameter": {
                          "accessId": "",
                          "accessKey": "",
                          "column": [],
                          "odpsServer": "",
                          "partition": "",
                          "project": "",
                          "table": "",
                          "truncate": true
                      }
                  }
              }
          ],
          "setting": {
              "speed": {
                  "channel": ""
              }
          }
      }
  }   

• 示例

  {
      "job": {
          "content": [
              {
                  "reader": {
                      "name": "mysqlreader",
                      "parameter": {
                          "username": "****",
                          "password": "****",
                          "column": ["id","age","name"],
                          "connection": [
                              {
                                  "table": [
                                      "test_table"
                                  ],
                                  "jdbcUrl": [
                                      "jdbc:mysql://127.0.0.1:3306/test"
                                  ]
                              }
                          ]
                      }
                  },
                  "writer": {
                      "name": "odpswriter",
                      "parameter": {
                          "accessId": "****",
                          "accessKey": "****",
                          "column": ["id","age","name"],
                          "odpsServer": "http://service.odps.aliyun.com/api",
                          "partition": "pt='datax_test'",
                          "project": "datax_opensource",
                          "table": "datax_opensource_test",
                          "truncate": true
                      }
                  }
              }
          ],
          "setting": {
              "speed": {
                  "channel": 1
              }
          }
      }
  }

• 第三步：启动DataX

  cd ../bin

   python datax.py ../job/job.json

  完成