最近由于工作需要写了一个处理数据的脚本,有几个用法在此记录一下。下面是代码
#!/bin/bash
if [[ -z $userDir ]]
then
export userDir="/tmp/"
fi
if [[ -z $spotDir ]]
then
export spotDir="/data/faust/data/invent_lac/removespot/spot/"
fi
USER="root"
PASSWORD="hgkf.8888"
DATABASE="tourism"
HOST="192.168.100.11"
function hand_time()
{
currentStamp=`date +%s`
lastStamp=$((currentStamp-24*3600))
standard=`date -d @"$lastStamp" +"%Y%m%d"`
standardTime=`date -d @"$lastStamp" +"%Y-%m-%d"`
standardTime="$standardTime 00:00:00"
usableStamp=`date -d "$standardTime" +%s`
#echo $standard
#echo $standardTime
#echo $usableStamp
cat ${spotDir}spot_$standard*|awk -F "," '{print$1}'|sort -u|sed 's/^/0,/g'|sed 's/$/',$usableStamp'/g' >${userDir}alluser.csv
cat ${spotDir}spot_$standard*|awk -F "," '{print$2","$1}'|sort -u |sed 's/$/',$usableStamp'/g' >${userDir}scenic.csv
}
function hand_user_data()
{
hand_time
allSql="load data infile '${userDir}alluser.csv' into table to_user_data fields terminated by ',' optionally enclosed by '\\\"' escaped by '\\\"' lines terminated by '\n' (id_marker,phone,ctime);"
scenicSql="load data infile '${userDir}scenic.csv' into table to_user_data fields terminated by ',' optionally enclosed by '\\\"' escaped by '\\\"' lines terminated by '\n' (id_marker,phone,ctime);"
scp ${userDir}alluser.csv ${userDir}scenic.csv 192.168.100.11:/tmp/ >${userDir}data.log
ssh -T 192.168.100.11 <
mysql -u$USER -p$PASSWORD $DATABASE -e "${allSql}"
mysql -u$USER -p$PASSWORD $DATABASE -e "${scenicSql}"
exit
EOF
}
hand_user_data
#hand_time
解析:
ssh -T 192.168.100.11 < mysql -u$USER -p$PASSWORD $DATABASE -e "${allSql}"
mysql -u$USER -p$PASSWORD $DATABASE -e "${scenicSql}"
exit
EOF
中注意
1,ssh -T 加-T的原因是告诉电脑我是在用shell脚本远程登录不打印登录信息
2,第二个EOF一定要在行首,不能有空格或tab键,否则会有警告
cat ${spotDir}spot_$standard*|awk -F "," '{print$1}'|sort -u|sed 's/^/0,/g'|sed 's/$/',$usableStamp'/g' >${userDir}alluser.csv
此句中sort -u排序后去重
sed用法总结
sed '1,1'd 删除第一行
sed -n '1,3p' 显示1~3行
sed 's/^/0,/g' 在行首加上0,
sed 's/$/',$usableStamp' 在行尾加上usableStamp的内容
sed 's/\t/,/g‘ 将‘\t’转换成“,”,此句在处理mysql查询结果时经常用到