Linux下Shell编程实现基于Hadoop的ETL(Sqoop封装篇)

conf/sqoop.xml

	
		jdbc:mysql://master:3306/db_cda
		root
		123
		
		
		
		
		
		
		
		
		
		1
		
		
		
		
		'\\\\N'
		'\\\\N'
		
		
		
		true
		
		
		true
		true
	
	
	
		jdbc:mysql://master:3306/db_cda
		root
		123
		
		
		1
		
		
		
		
		'\\t'
		
		
		
		
		
		'\\n'
		
		
	

bin/sqoop.sh

#!/bin/bash
#get the environment
if [ -f ~/.bashrc ];#进入Bash Shell环境，用于下面命令执行
then
 . ~/.bashrc

#获取当前命令的路径
bin=`dirname $0`
bin=`cd "$bin";pwd`
filename="$bin"/../conf/sqoop.xml
#if file not exsist,will exit;
fi
if [ -z $filename ];then
	echo 'USAGE:COMMAND FILENAME'
	exit 0
fi

#各种预定义变量及数组
declare -i i=0
declare -i j=0
declare -a importKeys
declare -a importValues
declare -a exportKeys
declare -a exportValues

flag=0
#处理掉配置文件中的空格、空行、以及注释等
content=`sed -e 's/\s*\(.*\)\s*$/\1/g' -e 's/\s*\(=\)\s*/\1/g' -e '/^\(\s\)*$/d' -e '/^$/d' -e 's///' $filename`
#然后循环遍历XML文件中的每一行
while read line
do \
	if echo ${line}|grep -qE "^" ; then
		flag=1
		continue
	fi

	if echo ${line}|grep -qE "^" ; then
		flag=2
		continue
	fi

	if echo ${line}|grep -qE "<\/sqoop-shell>" ; then
		flag=0
		continue
	fi

	if [ ${flag} -ne 0 ] ; then
		#处理获得到key值
		key=`echo ${line}|sed -n -e 's/.*<\/param>//p'|tr -d '\r'`
		#处理获得到Value值
		value=`echo ${line}|sed -n -e 's///' -e 's/<\/param>//p'|tr -d '\r'`
		#如果key值为空，则跳过
		if [ ${#key} -eq 0 ] ; then
			continue
		fi
		#否则根据标记，把key和value加入到导入或者导出数组中
		if [ ${flag} -eq 1 ] ; then
			importKeys[$i]=$key
			importValues[$i]=$value
			let i++
		elif [ ${flag} -eq 2 ] ; then
			exportKeys[$j]=$key
			exportValues[$j]=$value
			let j++
		fi
	fi
done \
<

具体Shell代码下载:http://download.csdn.net/detail/luo849278597/9490920

加群：397706991，共同学习