Azkaban：定时调度系统的使用
一：准备好需要使用到的脚本
1.sqoop import 拉取数据，如
#!/bin/bash
sqoop import --connect "jdbc:sqlserver://192.168.11.11;databaseName=AIM_FI;username=***;password=2***" --table Achieve --driver com.microsoft.sqlserver.jdbc.SQLServerDriver --hive-import --hive-table ods.ods_AIM_FIN_Achieve --hive-overwrite --hive-drop-import-delims --delete-target-dir -m 1
sqoop import --connect "jdbc:sqlserver://192.168.11.12;databaseName=WAL_FI;username=***;password=2***" --table Achieve --driver com.microsoft.sqlserver.jdbc.SQLServerDriver --hive-import --hive-table ods.ods_WAL_FIN_Achieve --hive-overwrite --hive-drop-import-delims --delete-target-dir -m 1
sqoop import --connect "jdbc:sqlserver://192.168.11.13;databaseName=ANL_FI;username=***;password=2***" --table Achieve --driver com.microsoft.sqlserver.jdbc.SQLServerDriver --hive-import --hive-table ods.ods_ANL_FIN_Achieve--hive-overwrite --hive-drop-import-delims --delete-target-dir -m 1 2.hive中执行的sql脚本
注意sql脚本中，每句sql需要分号分隔开，无用语句记得删掉，按次序执行，如下示例：
insert overwrite table ods.ods_aim_wal_anl_base_regionhistories
select * from ods.ods_aim_base_regionhistories;
insert into table ods.ods_aim_wal_anl_base_regionhistories
select * from ods.ods_wal_base_regionhistories;
insert into table ods.ods_aim_wal_anl_base_regionhistories
select * from ods.ods_anl_base_regionhistories;
insert overwrite table ods.ods_resumeachieve_t_regionhistories_tmp1
select ChargeName, regexp_replace(regexp_replace(sub, '\\{', ''), '\\}', '') managebranch,StartDate,chargeno from (select * from ods.ods_aim_wal_anl_base_regionhistories lateral view explode(split(ManageBranchNos,'\\}\\{')) tmp as sub) tmp_a;
insert overwrite table ods.ods_aim_wal_anl_base_employees
select * from ods.ods_aim_base_employees;
insert into table ods.ods_aim_wal_anl_base_employees
select * from ods.ods_wal_base_employees;
insert into table ods.ods_aim_wal_anl_base_employees
select * from ods.ods_anl_base_employees;
insert overwrite table ods.ods_aim_wal_anl_base_branchs
select * from ods.ods_aim_base_branchs;
insert into table ods.ods_aim_wal_anl_base_branchs
select * from ods.ods_wal_base_branchs;
insert into table ods.ods_aim_wal_anl_base_branchs
select * from ods.ods_anl_base_branchs;
insert overwrite table ods.ods_aim_wal_anl_base_teams
select * from ods.ods_aim_base_teams;
insert into table ods.ods_aim_wal_anl_base_teams
select * from ods.ods_wal_base_teams;
insert into table ods.ods_aim_wal_anl_base_teams
select * from ods.ods_anl_base_teams;
INSERT
overwrite TABLE dws.dws_resumeachieve_t_result_tmp1
SELECT
B.RecomId,
A.EffectiveDate,
A.ParentId,
A.CAID,
A.ResumeName,
A.Amount,
O.workno,
O.email,
O.isleave,
O.leavedate,
O.branchno,
O.teamsno,
CASE
WHEN instr(O.branchno,'AIM')>0
THEN '埃'
WHEN instr(O.branchno,'WAL')>0
THEN '沃'
WHEN instr(O.branchno,'ANL')>0
THEN '安'
ELSE O.branchno
END brand,
br.name office,
abt.name team,
D.IncomeIds,
B.AddTime,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '') IncomeIds0
,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '') IncomeIds1
,
oaer0.addtime AddTime0,
oaer1.addtime AddTime1,
O.fullname
FROM
ods.ods_AIM_FIN_AchieveAllots A
LEFT JOIN
ods.ods_aim_fin_achieves D
ON
A.ParentId=D.Id
LEFT JOIN
ods.ods_AIM_FIN_Payments B
ON
A.PaymentId=B.Id
LEFT JOIN
ods.ods_aim_erp_recommends C
ON
B.recomid=C.Id
LEFT JOIN
resume.ods_lieni_resume_t_cv_resume_info tcri
ON
A.CAID=tcri.id
LEFT JOIN
ods.ods_shanghai_lieni_t_gw_account tga
ON
tga.id=tcri.input_man_id
LEFT JOIN
ods.ods_aim_wal_anl_base_employees O
ON
O.email=tga.gw_email
LEFT JOIN
ods.ods_aim_wal_anl_base_branchs br
ON
br.number=O.branchno
LEFT JOIN
ods.ods_aim_wal_anl_base_teams abt
ON
abt.number=regexp_replace(regexp_replace(O.teamsno, '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_aim_fin_incomes oafi
ON
oafi.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_aim_erp_recommends oaer0
ON
oafi.recomid=oaer0.id
LEFT JOIN
ods.ods_aim_fin_incomes oafi1
ON
oafi1.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_aim_erp_recommends oaer1
ON
oafi1.recomid=oaer1.id
WHERE
A.branchno LIKE '%YUNYING'
AND Role='城市总经理'
AND (
A.Classify='A'
OR A.Classify='B')
AND D.IsCancel IS NULL
AND A.UnFinish IS NULL
AND A.IsDelete IS NULL
ORDER BY
B.RecomId;
INSERT
INTO
TABLE dws.dws_resumeachieve_t_result_tmp1
SELECT
B.RecomId,
A.EffectiveDate,
A.ParentId,
A.CAID,
A.ResumeName,
A.Amount,
O.workno,
O.email,
O.isleave,
O.leavedate,
O.branchno,
O.teamsno,
CASE
WHEN instr(O.branchno,'AIM')>0
THEN '埃'
WHEN instr(O.branchno,'WAL')>0
THEN '沃'
WHEN instr(O.branchno,'ANL')>0
THEN '安'
ELSE O.branchno
END brand,
br.name office,
abt.name team,
D.IncomeIds,
B.AddTime,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '') IncomeIds0
,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '') IncomeIds1
,
oaer0.addtime AddTime0,
oaer1.addtime AddTime1,
O.fullname
FROM
ods.ods_WAL_FIN_AchieveAllots A
LEFT JOIN
ods.ods_wal_fin_achieves D
ON
A.ParentId=D.Id
LEFT JOIN
ods.ods_wal_payments B
ON
A.PaymentId=B.Id
LEFT JOIN
ods.ods_wal_erp_recommends C
ON
B.recomid=C.Id
LEFT JOIN
resume.ods_lieni_resume_t_cv_resume_info tcri
ON
A.CAID=tcri.id
LEFT JOIN
ods.ods_shanghai_lieni_t_gw_account tga
ON
tga.id=tcri.input_man_id
LEFT JOIN
ods.ods_aim_wal_anl_base_employees O
ON
O.email=tga.gw_email
LEFT JOIN
ods.ods_aim_wal_anl_base_branchs br
ON
br.number=O.branchno
LEFT JOIN
ods.ods_aim_wal_anl_base_teams abt
ON
abt.number=regexp_replace(regexp_replace(O.teamsno, '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_wal_fin_incomes oafi
ON
oafi.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_wal_erp_recommends oaer0
ON
oafi.recomid=oaer0.id
LEFT JOIN
ods.ods_wal_fin_incomes oafi1
ON
oafi1.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_wal_erp_recommends oaer1
ON
oafi1.recomid=oaer1.id
WHERE
A.branchno LIKE '%YUNYING'
AND Role='城市总经理'
AND (
A.Classify='A'
OR A.Classify='B')
AND D.IsCancel IS NULL
AND A.UnFinish IS NULL
AND A.IsDelete IS NULL
ORDER BY
B.RecomId;
INSERT
INTO
TABLE dws.dws_resumeachieve_t_result_tmp1
SELECT
B.RecomId,
A.EffectiveDate,
A.ParentId,
A.CAID,
A.ResumeName,
A.Amount,
O.workno,
O.email,
O.isleave,
O.leavedate,
O.branchno,
O.teamsno,
CASE
WHEN instr(O.branchno,'AIM')>0
THEN '埃'
WHEN instr(O.branchno,'WAL')>0
THEN '沃'
WHEN instr(O.branchno,'ANL')>0
THEN '安'
ELSE O.branchno
END brand,
br.name office,
abt.name team,
D.IncomeIds,
B.AddTime,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '') IncomeIds0
,
regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '') IncomeIds1
,
oaer0.addtime AddTime0,
oaer1.addtime AddTime1,
O.fullname
FROM
ods.ods_ANL_FIN_AchieveAllots A
LEFT JOIN
ods.ods_anl_fin_achieves D
ON
A.ParentId=D.Id
LEFT JOIN
ods.ods_anl_payments B
ON
A.PaymentId=B.Id
LEFT JOIN
ods.ods_anl_erp_recommends C
ON
B.recomid=C.Id
LEFT JOIN
resume.ods_lieni_resume_t_cv_resume_info tcri
ON
A.CAID=tcri.id
LEFT JOIN
ods.ods_shanghai_lieni_t_gw_account tga
ON
tga.id=tcri.input_man_id
LEFT JOIN
ods.ods_aim_wal_anl_base_employees O
ON
O.email=tga.gw_email
LEFT JOIN
ods.ods_aim_wal_anl_base_branchs br
ON
br.number=O.branchno
LEFT JOIN
ods.ods_aim_wal_anl_base_teams abt
ON
abt.number=regexp_replace(regexp_replace(O.teamsno, '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_anl_fin_incomes oafi
ON
oafi.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[0], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_anl_erp_recommends oaer0
ON
oafi.recomid=oaer0.id
LEFT JOIN
ods.ods_anl_fin_incomes oafi1
ON
oafi1.id=regexp_replace(regexp_replace(split(D.IncomeIds,'\\}\\{')[1], '\\{', ''), '\\}', '')
LEFT JOIN
ods.ods_anl_erp_recommends oaer1
ON
oafi1.recomid=oaer1.id
WHERE
A.branchno LIKE '%YUNYING'
AND Role='城市总经理'
AND (
A.Classify='A'
OR A.Classify='B')
AND D.IsCancel IS NULL
AND A.UnFinish IS NULL
AND A.IsDelete IS NULL
ORDER BY
B.RecomId;
insert overwrite table dws.dws_resumeachieve_t_result_tmp2
select
recomId,
effectivedate,
ParentId,
CAID,
ResumeName,
amount,
--workno,
CASE
WHEN email is not null
THEN workno
ELSE ''
END workno,
email,
isleave,
leavedate,
branchno,
teamsno,
brand,
office,
team,
incomeids,
CASE
WHEN length(addtime)>0
THEN addtime
WHEN length(addtime0)>0
THEN addtime0
WHEN length(addtime1)>0
THEN addtime1
ELSE addtime
END addtime,
incomeids0,
incomeids1,
CASE
WHEN email is not null
THEN fullname
ELSE '未分配业绩'
END fullname
from dws.dws_resumeachieve_t_result_tmp1 tmp1;
insert overwrite table dws.dws_resumeachieve_t_result_tmp3
SELECT
*
FROM
(
SELECT
recomId,
effectivedate,
ParentId,
CAID,
ResumeName,
amount,
workno,
email,
isleave,
leavedate,
branchno,
teamsno,
brand,
office,
team,
incomeids,
addtime,
incomeids0,
incomeids1,
chargename,
chargeno,
startdate,
fullname,
row_number() over(partition BY recomId,effectivedate,ParentId,CAID,ResumeName,amount,workno,email,isleave,leavedate,branchno,teamsno,brand,office,team,incomeids ORDER BY startdate DESC) AS rank
FROM
(
SELECT
*
FROM
dws.dws_resumeachieve_t_result_tmp2 tmp2
LEFT JOIN
ods.ods_resumeachieve_t_regionhistories_tmp1 tmp_a
ON
tmp2.branchno=tmp_a.managebranch
) a ) tmp
WHERE
rank is null or rank=1;
insert overwrite table dws.dws_resumeachieve_t_result_last
select
CASE
WHEN isleave='true' and addtime>startdate
THEN chargeno
ELSE workno
END workno,
CASE
WHEN isleave='true' and addtime>startdate
THEN chargename
ELSE fullname
END employeename,
CASE
WHEN brand is not null
THEN brand
ELSE ''
END brand,
CASE
WHEN office is not null
THEN office
ELSE ''
END office,
CASE
WHEN team is not null
THEN team
ELSE ''
END team,
round(amount,1),
effectivedate
from dws.dws_resumeachieve_t_result_tmp3;
3.sqoop export数据到mysql：
#!/bin/bash
sqoop eval --connect "jdbc:mysql://172.16*.***.***:3309/p_b?useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&useSSL=true" \
--username p_i \
--password i*****pl \
--e "DELETE from t_cal_resume_achieve"
sqoop export \
--connect "jdbc:mysql://172.16*.***.***:3309/p_b?useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&useSSL=true" \
--username p_b\
--password i******pl \
--table t_cal_resume_achieve \
--update-key cno \
--update-mode allowinsert \
--input-fields-terminated-by "\001" \
--hcatalog-database dws \
--hcatalog-table dws_resumeachieve_t_result_last \
--columns="workno,employeename,brand,office,team,amount, effectivedate" \
--num-mappers 1 --input-null-string '\\N' --input-null-non-string '\\N'
如上3个脚本准备好后，在linux某目录下新建3个文件
sqoop_import.sh,hive_convert.sh,sqoop_export.sh
分别存放以上3个脚本内容，切记要粘贴到linux保存，如果直接上传这个文件到linux，容易出问题。
二：准备好Azkaban的job文件并打包，然后将任务上传到Azkaban
新建一个文件夹，名字自取，如achieve
新建3个job文件，内容如下：
sqoop_import.job：
#sqoop_import.job
type=command
command=/bin/bash /*****/sqoop_import.sh #hive_convert.job
type=command
dependencies=sqoop_import
command=hive -f /******/hive_convert.sh #sqoop_export.job
type=command
dependencies=hive_convert
command=/bin/bash /***/sqoop_export.sh 注意：井号 这里是必须有的，然后需要跟文件名对应上，command 就是执行的命令，后面是你放sh脚本的目录，还有，如果有依赖的话，加上上级的依赖 比如：sqoop_import.job 依赖的是hive_convert.job,hive_convert.job依赖的是sqoop_export.job
最后将这个文件夹（名字自取），打包zip，如achieve.zip
创建project：
上传任务：
三：定时配置调度任务
任务跑失败了还可以添加邮箱，给我们的邮箱发邮件：
然后点击左下角按钮Schedule 配置下任务跑的时间即可。
注意要开启发邮件功能，可能需要改源码等。
任务失败了，可以设置重试的次数，配置里添加参数retries：
其他时间设置：
每周日9点30执行(注意尽量在上面框里填写，下面会自动生成表达式)：
设置上午9点到晚上9点每小时跑一次：
参考：
https://www.wandouip.com/t5i241597/
https://blog.csdn.net/aizhenshi/article/details/80828726