Sqoop将MySQL数据增量导入Hive示例

Sqoop将MySQL数据增量导入Hive示例

#!/bin/bash

export sqoop_home=/usr/bin/sqoop

-- 判断有无传参，有传参使用传参
if [ $# -eq 1 ]
then 
   datestr=$1
else
  datestr=`date -d '-1 day' +%Y  -%m-%d`
fi

start_time=`date +%Y-%m-%d`
year_stat=`date +%Y`
month_stat=`date +%m` 

jdbc_url='jdbc:mysql://hadoop01:3306/nev'
username='root'
password='123456'


${sqoop_home} import \
--connect ${jdbc_url} \
--username ${username} --password ${password} \
--query "SELECT  
id,create_date_time,session_id,sid,create_time,seo_source,seo_keywords,
ip,area,country,province,city,origin_channel,user AS user_match,
manual_time,begin_time,end_time,last_customer_msg_time_stamp,
last_agent_msg_time_stamp,reply_msg_count,msg_count,browser_name,os_info,
'${start_time}' AS starts_time
FROM web_chat_ems_${year_stat}_${month_stat} 
WHERE create_time BETWEEN '${datestr} 00:00:00'  AND '${start_time} 00:00:00' and \$CONDITIONS" \
--hcatalog-database itcast_ods \
--hcatalog-table web_chat_ems \
--hive-partition-key starts_time \
--hive-partition-value ${datestr} \
-m 100 \
--split-by id


wait

${sqoop_home} import \
--connect ${jdbc_url} \
--username ${username} --password ${password} \
--query "SELECT 
wcte.id,
wcte.referrer,
wcte.from_url,
wcte.landing_page_url,
wcte.url_title,
wcte.platform_description,
wcte.other_params,
wcte.history,
'${start_time}' AS start_time
FROM
(SELECT * 
FROM web_chat_ems_${year_stat}_${month_stat}
WHERE create_time BETWEEN '${datestr} 00:00:00'  AND '${start_time} 00:00:00') tmp 
JOIN web_chat_text_ems_${year_stat}_${month_stat} wcte ON  tmp.id = wcte.id and \$CONDITIONS" \
--hcatalog-database itcast_ods \
--hcatalog-table web_chat_text_ems \
--hive-partition-key start_time \
--hive-partition-value ${datestr} \
-m 100 \
--split-by id