DataX导数据从mysql到hive回顾

太久没碰datax了，没想到真的忘记了
打开hdfs
start-all.sh
打开hive
hive --service metastore &
hive --service hiveservr2 &
hive
注意：一定要先开启meta，再开启hive！
mysql可以确定一下表
mysql -uroot -p'密码'

在hive创建表

create table if not exists test(
id int,
name string
)
row format delimited 
fields terminated by "\u001" 
stored as orc;

datax的使用

cd /usr/local/data/job
vim test01.json

案例

{ "job":{
    "setting":{
        "speed":{
            "channel":2
        }
    },
    "content":[
        {
            "reader":{
                "name":"mysqlreader",
                "parameter":{
                    "username":"root",
                    "password":"****",
                    "connection":[
                        {
                            "querySql":[
                                "select * from test"
                            ],
                            "jdbcUrl":[
                                "jdbc:mysql://qianfeng01:3306/sz2103"
                            ]
                        }
                    ]
                }
            },
            "writer":{
                "name":"hdfswriter",
                "parameter":{
                    "defaultFS":"hdfs://qianfeng01:8020",
                    "fileType":"orc",
                    "path":"/user/hive/warehouse/test.db/test",
                    "fileName":"test",
                    "column":[
                        {"name":"id","type":"int"}
                        ,{"name":"name","type":"string"}
                        
                    ],
                    "writeMode":"append",
                    "fieldDelimiter":"\u0001"
                }
            }
        }
    ]
}
}

命令
[root@qianfeng01 ~]# python /usr/local/datax/bin/datax.py /usr/local/datax/job/test01.json
结果

调度里面需要的代码
下面的20220909就是我之前的时间账期

hive -e "
alter table databasename.test add if not exists partition (date_no=20220909) ;
truncate table databasename.test partition (date_no=20220909);"
python /web/soft/datax/bin/datax.py -p "-DDate_No='20220909'" /web/soft/datax/job/ftp_to_hdfs/test_16.json
#如果ftp的日志服务器有多台，就是可以把json里面主机地址修改一下，复制多个json
python /web/soft/datax/bin/datax.py -p "-DDate_No='20220909'" /web/soft/datax/job/ftp_to_hdfs/test_17.json