webgame 日志解析的过程

awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log.2011-08-15-23  > tmp.1



awk 'BEGIN {FS="|"}; {print $1}' tmp.1 | awk 'BEGIN {FS=":"}; {print $1}' > tmp.2

合并

awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; {print $1 | awk 'BEGIN {FS=":"}; {print $1}  "|" $2 "|" $3}' > tmp.3

awk 'BEGIN {FS="|"}; [{print awk 'BEGIN {FS=":"}; {print $1}' > tmp.3

-----------------------------------

awk 'BEGIN {FS="|"}; {print ($1 || substr($1, 1, index($1,":")-1 )) "|" $2 "|" $3}' tmp.1 > tmp.4

awk 'BEGIN {FS="|"}; {print substr($1, 1, 9) "|" $2 "|" $3}' tmp.1 > tmp.4
----------------------------------------------------------------
awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' tmp.1 > tmp.4  OK

合并：

清理gamelogin.log数据,按时间分：
格式

23|255123768|mszj|1001

awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' > tmp.6

awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' tmp.10 | sort| uniq -c > tmp.7

------------------------------
合并：
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c | sort -r -n > tmp.7

逆序输出：
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c |awk 'BEGIN {FS=" "}; {print $3 " " $1 " " $5 " " $4 " " $2 }' | sort -r -n > tmp.7

+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
promitioin.log的处理

awk 'BEGIN {FS=","}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4 "|" $5 "|" $6  "|" $7 "|" $8 }' promition.log.2011-08-15-23 > log.1

awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $2 " " $3  " " $4 " " $5 " "  $8 " "substr($1,index($1, " ") + 1, 2) }' promition.log.2011-08-15-23 | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $2 " " $3 " " $1 " " $5 " " $4 " " $6 " " $7 " " $8 " " $9}'| sort -r -n >log.2

cat test.txt test1.txt | sort | uniq -d
vimdiff tmp1 tem2

cat tmp1 tmp2 | uniq -d

awk '{print $7}' log.2 | sort | uniq > log2.log
合并文件
cat file1.out file2.out ＞ file2.out 
cat promition.success.log.2011-08-15-* > promition.success.log

gamelogin.log 16日

awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3  "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c |awk 'BEGIN {FS=" "}; {print $3 " " $1 " " $5 " " $4 " " $2 }' | sort -r -n > tmp
-----------------------------------------------------------------------
awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log > tmp.1

awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' tmp.1 > tmp.2

awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' > tmp.3

awk 'BEGIN {FS="|"}; {print $1 " " $2 " " $3 }' tmp.3 | sort| uniq -c > tmp.4

awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5

awk '{ sub(/^[ \t]+/,""); print }' tmp.5 > tmp.7
awk 'gsub(/^ *| *$/,"")'

awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}' tmp.5  > tmp.6

sort tmp.5  > tmp.6
-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5

awk 'BEGIN {FS="|"}; {print $2}' tmp.5 | uniq | sort > tmp-uid

===================================================================
promition.success.log 15日

awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $2 " " $3  " " $4 " " $5 " "  $8 " "substr($1,index($1, " ") + 1, 2) }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $2 " " $3 " " $1 " " $5 " " $4 " " $6 " " $7 " " $8 " " $9}'| sort -r -n >log.1


awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort >log.1

awk 'BEGIN {FS="|"}; {print $2}' log.1 | uniq | sort > log-uid

//多个文件合并为一行：
// paste glogin-uid-count.log prom-suc-uid-count.log log-rate.log > flux.log
----------------------------------------------------------------------
算登录率
cat log-uid tmp-uid | sort | uniq -d|wc -l >test.log

最终整理如下：

awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log
awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid
awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid
计算前一天的次登录率
cat glogin-uid | wc -l  > tmp-glog-prom-uid.log
cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
单个文件合并为一行
计算登录率
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log
rm -f tmp*

-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log

awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid

awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log

awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid

cat glogin-uid | wc -l  > tmp-glog-prom-uid.log

cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log

cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log

cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log

awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log

rm -f tmp*

结果 532853|36842|15950|0.43293
----------------------------------
简化版
touch test.c 就创建了一个新文件 test.c

获取前一天的时间

cd /data/remote-log-server/log/promition/2011-08-17
cat promition.success.log.2011-08-17-* > promition.success.log.2011-08-17
scp promition.success.log.2011-08-17 [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/
rm -f promition.success.log.2011-08-17

cd /data/remote-log-server/log/gamelogin/2011-08-17
scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.2011-08-17


echo `date -d"300 days ago" +%Y-%m-%d`
------------------------------------------------
单日拷贝数据
echo "************start**************"
  dd=`date +%Y-%m-%d -d " 1 days ago " `

  cd /data/remote-log-server/log/promition/$dd
  cat promition.success.log.$dd-* > promition.success.log.$dd
  scp promition.success.log.$dd [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/
  rm -f promition.success.log.$dd
  echo "************ $dd promition.success end**************"
  
  cd /data/remote-log-server/log/gamelogin/$dd
  scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.$dd

  echo "************ $dd gamelogin end**************"

循环拷贝数据
10.3.17.26
/data/remote-log-server/run/wqo
sh testfor.log
k=0
while((k++<=2))
do
  echo "************start**************"
  dd=`date +%Y-%m-%d -d " $k days ago " `
  cd /data/remote-log-server/log/promition/$dd
  cat promition.success.log.$dd-* > promition.success.log.$dd
  scp promition.success.log.$dd [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/
  rm -f promition.success.log.$dd
  echo "************ $dd promition.success end**************"
  
  cd /data/remote-log-server/log/gamelogin/$dd
  scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.$dd

  echo "************ $dd gamelogin end**************"

done

--------------------------------------------

#!/bin/bash
# This script prints a message about your weight if you give it your
# weight in kilos and hight in centimeters.
if [ ! $# == 2 ]; then
  echo "Usage: $0 weight_in_kilos length_in_centimeters"
  exit
fi
weight="$1"
height="$2"
idealweight=$[$height - 110]
if [ $weight -le $idealweight ] ; then
  echo "You should eat a bit more fat."
else
  echo "You should eat a bit more fruit."
fi
# weight.sh 70 150
You should eat a bit more fruit.
# weight.sh 70 150 33

---------------------------------------------------------

awk '{FS="|"}; {print $6 "\t" $5 "\t" $4 "\t" $3 "\t" $2 "\t" $1 }' flux-rate.log | sort -k 1 > fff.log 

cat flux-rate.log | sort -k 1 > fff.log
cat flux-rate.log  | sort > fff.log 

ps aux | grep testfor.sh

---------------------------------------------

#!/bin/sh

echo "*********start********"

dd=`date +%Y-%m-%d -d " 1 days ago " `

if [ ! -f "gamelogin.log.$dd" ] ; then
    
   echo "*****gamelogin.log.$dd not existed*******"
   
   exit
fi
 

if [ ! -f "glogin.log.$dd" ] ; then    

   echo "***********glogin.log.$dd not exist, start to create it**************"

   awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd

   echo "***********glogin.log.$dd end***********"
 
else
   
   echo "************glogin.log.$dd is existed********************"

fi


awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd

cat glogin-uid.$dd | wc -l  > tmp-glog-prom-uid.log

################################################################

tt=`date +%Y-%m-%d -d " 2 days ago " `

if [ -f "promition.success.log.$tt" ] ; then


    if [ ! -f "prom-suc.log.$tt" ] ; then

      echo "************prom-suc.log.$tt not exist , start to create it***********************"
  
      awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt

      echo "***********prom-suc.log.$tt end***************"

    else 
  
      echo "***********prom-suc.log.$tt is existed**********************"

    fi

    awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq  > prom-suc-uid.$tt


    cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log

    cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d | wc -l >> tmp-glog-prom-uid.log

    echo $dd >>tmp-glog-prom-uid.log

    echo $tt >>tmp-glog-prom-uid.log

    cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log

    awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2 "|" $4 "|" $5 }' tmp-flux-rate.log >> flux-rate.log

    echo "***********flux-rate.log end************"
    rm -f tmp*

else

    echo "********promition.success.log.$tt not exit!***************"

    exit

fi

echo "*****************end**********************"

-------------------------------------------------------------

#!/bin/sh

echo "*********start********"

m=0
while((m++<=30))
do

dd=`date +%Y-%m-%d -d " $m days ago " `

if [ ! -f "gamelogin.log.$dd" ] ; then
    
   echo "*****gamelogin.log.$dd not existed*******"
   
   exit
fi
 

if [ ! -f "glogin.log.$dd" ] ; then    

   echo "***********glogin.log.$dd not exist, start to create it**************"

   awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd

   echo "***********glogin.log.$dd end***********"
 
else
   
   echo "************glogin.log.$dd is existed********************"

fi

if [ ! -f "glogin-uid.$dd" ] ; then
   echo "********************glogin-uid.$dd not exist, create id***********"
   awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd

else
   echo "********************glogin-uid.$dd is existed******"
fi


################################################################

echo "****for promition*****"

k=0
while((k++<30))
do

((n=k+m))

tt=`date +%Y-%m-%d -d " $n  days ago " `

  if [ -f "promition.success.log.$tt" ] ; then


      if [ ! -f "prom-suc.log.$tt" ] ; then

        echo "************prom-suc.log.$tt not exist , start to create it***********************"
  
        awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt

        echo "***********prom-suc.log.$tt end***************"

      else 
  
        echo "***********prom-suc.log.$tt is existed**********************"

      fi
       
      if [ ! -f "prom-suc-uid.$tt" ] ; then
         
        echo "***************prom-suc-uid.$tt is not exited, create**********************"
        
        awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt

      else
     
        echo "****************prom-suc-uid.$tt is existed********************"

      fi

      cat glogin-uid.$dd | wc -l  > tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d | wc -l >> tmp-glog-prom-uid.log

      echo $dd >>tmp-glog-prom-uid.log

      echo $tt >>tmp-glog-prom-uid.log

      cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log

      awk 'BEGIN {FS=" "}; {print $5 "|" $4 "|" $3/$2 "|" $3 "|" $2 "|" $1 }' tmp-flux-rate.log >> flux-rate.log

      echo "***********flux-rate.log end************"
      rm -f tmp*

  else

      echo "********promition.success.log.$tt not exit!***************"

  fi


done

done

echo "*****************end**********************"