webgame 日志脚本-最终版
0、testlogfor.sh
1、test.sh
2、testfor.sh
3、testcatfor.sh
3、testservicefor.sh
4、python_webgame.py
已可从第一行读csv
#!/bin/sh echo `date` export PATH="/usr/kerberos/bin:$PATH" export KRB5CCNAME=/tmp/krb5cc_pub_$$ trap kdestroy 0 1 2 3 5 15 kinit -k -t /etc/krb5.keytab k=0 while((k++<1)) do dd=`date +%Y-%m-%d -d " $k days ago"` echo $dd ssh [email protected] "cat /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd}-* > /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd};" scp [email protected]:/data/remote-log-server/log/promition/$dd/promition.success.log.${dd} /data/remote-log-server/log/gamelogin/wqo/test/promition.success.log.${dd} ssh [email protected] "rm -f /data/remote-log-server/log/promition/$dd/promition.success.log.${dd};" echo "************ $dd promition.success end**************" scp [email protected]:/data/remote-log-server/log/gamelogin/${dd}/gamelogin.log /data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.${dd} echo "************ $dd gamelogin end**************" done
1、test.sh
#!/bin/sh
cd /data/remote-log-server/log/gamelogin/wqo/test
if [ -f "flux-rate.log" ] ; then
echo "*****flux-rate.log is existed**********"
> flux-rate.log
fi
echo "*********start********"
m=0
while((m++<1))
do
##dd=`date +%Y-%m-%d -d " $m days ago " `
dd="2011-08-24"
if [ ! -f "gamelogin.log.$dd" ] ; then
echo "*****gamelogin.log.$dd not existed*******"
exit
fi
if [ ! -f "glogin.log.$dd" ] ; then
echo "***********glogin.log.$dd not exist, start to create it**************"
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd
echo "***********glogin.log.$dd end***********"
else
echo "************glogin.log.$dd is existed********************"
fi
if [ ! -f "glogin-uid.$dd" ] ; then
echo "********************glogin-uid.$dd not exist, create id***********"
awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd
else
echo "********************glogin-uid.$dd is existed******"
fi
################################################################
echo "****for promition*****"
k=0
while((k++<1))
do
((n=k+m))
##tt=`date +%Y-%m-%d -d " $n days ago " `
tt="2011-08-10"
if [ -f "promition.success.log.$tt" ] ; then
if [ ! -f "prom-suc.log.$tt" ] ; then
echo "************prom-suc.log.$tt not exist , start to create it***********************"
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt
echo "***********prom-suc.log.$tt end***************"
else
echo "***********prom-suc.log.$tt is existed**********************"
fi
if [ ! -f "prom-suc-uid.$tt" ] ; then
echo "***************prom-suc-uid.$tt is not exited, create**********************"
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt
else
echo "****************prom-suc-uid.$tt is existed********************"
fi
cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log
cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log
cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log
echo $dd >>tmp-glog-prom-uid.log
echo $tt >>tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log
echo "***********flux-rate.log end************"
rm -f tmp*
else
echo "********promition.success.log.$tt not exit!***************"
fi
done
done
##cat flux-rate.log | sort > webgame-total.csv
echo "*****************total end**********************"
echo "****************category start*****************************"
##sh testcatfor.sh
echo "****************category end ******************************"
echo "***************service start ******************************"
##sh testservicefor.sh
echo "***************service end *****************************"
echo "****************start insert data into db*****************************"
##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv
##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv
##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csv
echo "****************data end*********************"
2、testfor.sh
#!/bin/sh
start=`date`
echo $start
cd /data/remote-log-server/log/gamelogin/wqo/test
if [ -f "flux-rate.log" ] ; then
echo "*****flux-rate.log is existed**********"
> flux-rate.log
fi
echo "*********start********"
m=0
while((m++<1))
do
dd=`date +%Y-%m-%d -d " $m days ago " `
if [ ! -f "gamelogin.log.$dd" ] ; then
echo "*****gamelogin.log.$dd not existed*******"
exit
fi
if [ ! -f "glogin.log.$dd" ] ; then
echo "***********glogin.log.$dd not exist, start to create it**************"
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd
echo "***********glogin.log.$dd end***********"
else
echo "************glogin.log.$dd is existed********************"
fi
if [ ! -f "glogin-uid.$dd" ] ; then
echo "********************glogin-uid.$dd not exist, create id***********"
awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd
else
echo "********************glogin-uid.$dd is existed******"
fi
################################################################
echo "****for promition*****"
k=0
while((k++<30))
do
((n=k+m))
tt=`date +%Y-%m-%d -d " $n days ago " `
if [ -f "promition.success.log.$tt" ] ; then
if [ ! -f "prom-suc.log.$tt" ] ; then
echo "************prom-suc.log.$tt not exist , start to create it***********************"
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt
echo "***********prom-suc.log.$tt end***************"
else
echo "***********prom-suc.log.$tt is existed**********************"
fi
if [ ! -f "prom-suc-uid.$tt" ] ; then
echo "***************prom-suc-uid.$tt is not exited, create**********************"
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt
else
echo "****************prom-suc-uid.$tt is existed********************"
fi
cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log
cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log
cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log
echo $dd >>tmp-glog-prom-uid.log
echo $tt >>tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log
echo "***********flux-rate.log end************"
rm -f tmp*
else
echo "********promition.success.log.$tt not exit!***************"
fi
done
done
cat flux-rate.log | sort > webgame-total.csv
echo "*****************total end**********************"
echo "****************category start*****************************"
sh testcatfor.sh
echo "****************category end ******************************"
echo "***************service start ******************************"
sh testservicefor.sh
echo "***************service end *****************************"
echo "****************start insert data into db*****************************"
/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv
/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv
/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csv
echo "****************data end*********************"
end=`date`
echo $end
3、testcatfor.sh
#!/bin/sh
if [ -f "flux-cat-rate.log" ]; then
echo "****flux-cat-rate.log is existed******"
> flux-cat-rate.log
fi
echo "**********analysis category and service number**************"
n=0
while((n++<1))
do
tt=`date +%Y-%m-%d -d " $n days ago" `
if [ ! -f "glogin.log.$tt" ] ; then
echo "********glogin.log.$tt not existed********************"
fi
echo "****glogin.log.$tt is existed********"
m=0
while((m++<=30))
do
((p=m+n))
dd=`date +%Y-%m-%d -d " $p days ago " `
if [ ! -f "prom-suc.log.$dd" ] ; then
echo "********prom-suc.log.$dd not existed****************"
else
echo "***********prom-suc.log.$dd is existed*******************"
awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log
awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log
catnum=`cat tmp-pcat-name.log | wc -l`
echo "**$catnum**"
#####for######
k=0
while((k++<${catnum}))
do
catcontent=`sed -n "${k} p" tmp-pcat-name.log`
echo "---------$catcontent------------"
cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log
awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-uid.log
####tt=`date +%Y-%m-%d -d " 1 days ago" `
cat glogin.log.$tt | grep $catcontent > tmp-glogin-cat.log
awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-glogin-cat.log | sort | uniq > tmp-glogin-cat-uid.log
cat tmp-prom-suc-cat-uid.log | wc -l > tmp-cat-prom-glogin-uid.log
cat tmp-glogin-cat-uid.log | wc -l >> tmp-cat-prom-glogin-uid.log
cat tmp-prom-suc-cat-uid.log tmp-glogin-cat-uid.log | sort | uniq -d | wc -l >> tmp-cat-prom-glogin-uid.log
echo $catcontent >> tmp-cat-prom-glogin-uid.log
echo $dd >> tmp-cat-prom-glogin-uid.log
echo $tt >> tmp-cat-prom-glogin-uid.log
cat tmp-cat-prom-glogin-uid.log | xargs echo > tmp-cat-rate.log
awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-cat-rate.log >> flux-cat-rate.log
done
echo "**********flux-cat-rate.log***************"
fi
done
done
cat flux-cat-rate.log | sort > webgame-cat.csv
3、testservicefor.sh
#!/bin/sh
if [ -f "flux-service-rate.log" ]; then
echo "****flux-service-rate.log is existed******"
> flux-service-rate.log
fi
echo "**********analysis category and service number**************"
n=0
while((n++<1))
do
tt=`date +%Y-%m-%d -d " $n days ago" `
if [ ! -f "glogin.log.$tt" ] ; then
echo "********glogin.log.$tt not existed********************"
fi
echo "****glogin.log.$tt is existed********"
m=0
while((m++<30))
do
((p=m+n))
dd=`date +%Y-%m-%d -d " $p days ago " `
if [ ! -f "prom-suc.log.$dd" ] ; then
echo "********prom-suc.log.$dd not existed****************"
else
echo "***********prom-suc.log.$dd is existed*******************"
awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log
awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log
catnum=`cat tmp-pcat-name.log | wc -l`
echo "**$catnum**"
#####for######
k=0
while((k++<${catnum}))
do
catcontent=`sed -n "${k} p" tmp-pcat-name.log`
echo "---------$catcontent------------"
cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-service-uid.log
awk 'BEGIN {FS="|"}; {print $3}' tmp-prom-suc-cat-service-uid.log | sort | uniq > tmp-pcat-service-name.log
servicenum=`cat tmp-pcat-service-name.log | wc -l`
echo "+++++$servicenum++"
######for service##########
g=0
while((g++<${servicenum}))
do
servicecontent=`sed -n "${g} p" tmp-pcat-service-name.log`
echo "---------------$servicecontent------------"
cat glogin.log.$tt | grep $catcontent > tmp-glogin-catser.log
###### echo "*****11111********"
cat tmp-glogin-catser.log | grep $servicecontent > tmp-glogin-cat-service.log
##### echo "*****22222*********"
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4 }' tmp-glogin-cat-service.log | sort | uniq > tmp-glogin-cat-service-uid.log
cat tmp-prom-suc-cat-service-uid.log | grep $servicecontent | wc -l > tmp-service-prom-glogin-uid.log
cat tmp-glogin-cat-service-uid.log | wc -l >> tmp-service-prom-glogin-uid.log
cat tmp-prom-suc-cat-service-uid.log tmp-glogin-cat-service-uid.log | sort | uniq -d | wc -l >> tmp-service-prom-glogin-uid.log
echo $catcontent >> tmp-service-prom-glogin-uid.log
echo $servicecontent >> tmp-service-prom-glogin-uid.log
echo $dd >> tmp-service-prom-glogin-uid.log
echo $tt >> tmp-service-prom-glogin-uid.log
cat tmp-service-prom-glogin-uid.log | xargs echo > tmp-service-rate.log
awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $7 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-service-rate.log >> flux-service-rate.log
done
done
echo "**********flux-service-rate.log***************"
fi
done
done
cat flux-service-rate.log | sort > webgame-cat-service.csv
4、python_webgame.py
已可从第一行读csv
#!/usr/bin/python
#python_webgame.py
import sys, csv
import MySQLdb
def main(user, pwd, db, table, csvfile):
try:
conn = getconn(user, pwd, db)
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit(1)
cursor = conn.cursor()
loadcsv(cursor, table, csvfile)
cursor.close()
conn.close()
def getconn(user, pwd, db):
conn = MySQLdb.connect(host = "10.3.18.158",
user = user,
passwd = pwd,
db = db)
return conn
def nullify(L):
"""Convert empty strings in the given list to None."""
#helper function
def f(x):
if(x == ""):
return None
else:
return x
return [ f(x) for x in L ]
def loadcsv(cursor, table, filename):
"""
Open a csc file and load it into a sql table.
Assumptions:
- the first line in the file is a header
"""
f = open(filename)
c = csv.reader(f)
header = c.next()
numfields = len(header)
query = buildInsertCmd(table, numfields)
f.seek(0)
for line in c:
if len(line)<1: continue
vals = nullify(line)
cursor.execute(query, vals)
return
def buildInsertCmd(table, numfields):
"""
example:
>>>buildInserCmd("foo", 3)
'insert int foo values (%s,%s,%s)'
"""
assert(numfields > 0)
placeholders = (numfields-1) * "%s, " + "%s"
query = ("insert into %s" % table) + (" values (null, %s)" % placeholders)
return query
if __name__ == '__main__':
args = sys.argv[1:]
if(len(args) < 5):
print "error: arguments: user \"password\" db table csvfile"
sys.exit(1)
main(*args)