oracle后台导入海量级数据

工作路径:/u02/work

准备工作:

1.控制文件:gen_ctl.sh

#************************************************************************************
#*****************gen_ctl.sh TDS Lzw2009
#*****************generate ctl file from a table.
#*****************Usage: gen_ctl.sh <userid/passwd[@connection]> <table_name>
#************************************************************************************

userid=$1
table=$2

lv_temp="wk_${table}.test"
lv_temp1="wk_${table}.test1"
lv_temp2="wk_${table}.test2"
lv_control="${table}.ctl"

sqlplus ${userid} <<!
spool ${lv_temp};
desc ${table}
spool off;
exit
!
if [ "$?" -ne 0 ]
then
echo "Error:sqlplus ${userid} error in generate control file for table ${table} !"
echo "please check userid and passwd or oracle_sid."
exit
fi

if [ -f ${lv_temp} ]
then
lv_line_num=`cat ${lv_temp}|grep -v "^SQL>" |grep -v " Name " |grep -v " -------" |wc -l`
lv_line_num=`expr ${lv_line_num} - 1`
lvstr="{if (/$2==/"DATE/") { if (NR<${lv_line_num}) {print /$1,/$2,/"///"MM/DD/YYYY///",/"}else{print /$1,/$2,/"///"MM/DD/YYYY///"/"}} else {if (NR<${lv_line_num}) {print /$1/",/"} else{print /$1}}}"
echo ${lvstr} > b.awk
cat ${lv_temp}|grep -v "^SQL>" |grep -v " Name " |grep -v " -------" |awk -f b.awk > ${lv_temp1}
else
echo "$0 error :not find ${lv_temp} file."
exit
fi

lv_str="LOAD DATA INFILE '${table}.unl' BADFILE 'bad_${table}.bad' APPEND INTO TABLE ${table} FIELDS TERMINATEd BY /"|/""
echo ${lv_str} > ${lv_control}
echo "(" >> ${lv_control}
cat ${lv_temp1} >> ${lv_control}
echo ")" >> ${lv_control}

rm -f ${lv_temp}
rm -f ${lv_temp1}
2.导数文件:cat gen_sh.sh
#************************************************************************************
#*****************gen_sh.sh TDS Lzw2009
#*****************generate shell file to load file. column sep is '|'
#*****************Usage: gen_sh.sh <userid/passwd[@connection]> <table_name>
#************************************************************************************
if [ ! $# -eq 2 ]
then
echo "Usage: $0 <userid/passwd[@connection]> <table_name>"
exit
else
userid=$1
table=$2
fi

lv_rows=10000
lv_bindsize=8192000
lv_readsize=8192000

echo "sqlldr ${userid} control=${table}.ctl rows=${lv_rows} bindsize=${lv_bindsize} readsize=${lv_readsize} log=log_${table}.log bad=bad_${table}.bad direct=true" > load_${table}.sh

执行工作:

先创建表LZW_TMP_CZ081112:

create table LZW_TMP_CZ081112
(
serv varchar2(11),
brand varchar2(10),
fee varchar2(10),
c_time varchar2(20),
c_type varchar2(10)
);

mv 20081112cz.txt LZW_TMP_CZ081112.unl

sh gen_ctl.sh "work/password@zjana" LZW_TMP_CZ081112

生成LZW_TMP_CZ081112.ctl文件

vi LZW_TMP_CZ081112.ctl

把列这间的分隔符改成','(逗号)

sh gen_sh.sh work/password@zjana LZW_TMP_CZ081112

生成load_LZW_TMP_CZ081112.sh文件

最后运行:sh load_LZW_TMP_CZ081112.sh即可

执行统计脚本:

select brand,
sum(case when fee ='30' then 1 else 0 end ) f3,
sum(case when fee ='50' then 1 else 0 end ) f5,
sum(case when fee ='100' then 1 else 0 end ) f10,
sum(case when fee ='300' then 1 else 0 end ) f30,
c_time,c_type
from LZW_TMP_CZ081112
group by brand,c_time,c_type;

select brand,c_time,
sum(case when c_type ='实充' then f3 else 0 end)f31,
sum(case when c_type =''实充' 'then f5 else 0 end)f51,
sum(case when c_type =''实充' 'then f10 else 0 end) f101,
sum(case when c_type =''实充' 'then f30 else 0 end) f301,
sum(case when c_type ='空充'then f3 else 0 end) f32,
sum(case when c_type =空充'then f5 else 0 end) f52,
sum(case when c_type ='空充'then f10 else 0 end) f102,
sum(case when c_type =空充'then f30 else 0 end) f302
from LZW_TMP_CZ081112bk