分析rails日志,计算响应率

chmod +x analyse_log copy_log

 

(1)拷贝脚本到本地

脚本内容如下:

#!/usr/bin/env bash

LOG_PATH=/opt/logs
LOCAL_LOG_PATH=~/Documents/logs/8-15
PROJECTS=(m-cms-for-tudou-tv m-cms-for-tv)

if [[ $1 == "--debug" ]]; then
  set -x
fi

# Create local directories
for project in ${projects[@]}; do
  mkdir -p $SAVE_TO/$project
done

# Copy m-cms-for-tv
project=m-cms-for-tv
remote_log_dir=$LOG_PATH/$project

for server in {13..15}; do
  for port in {5200..5207}; do
    server_ip="10.xxx.32.$server"
    local_log_dir=$LOCAL_LOG_PATH/$project/$server_ip
    log_file="thin server (0.0.0.0:$port)_2015-08-15.log"

    if [[ ! -e $local_log_dir/"$log_file" ]]; then
      mkdir -p $local_log_dir
      scp $server_ip:$remote_log_dir/"'$log_file'" $local_log_dir/$port.log
    fi
  done
done

# Copy m-cms-for-todou-tv
project=m-cms-for-tudou-tv
remote_log_dir=$LOG_PATH/$project

for server in {13..15}; do
  for port in {3201..3208}; do
    server_ip="10.xxx.32.$server"
    local_log_dir=$LOCAL_LOG_PATH/$project/$server_ip
    log_file="thin server (0.0.0.0:$port)_2015-08-15.log"

    if [[ ! -e $local_log_dir/"$log_file" ]]; then
      mkdir -p $local_log_dir
      scp $server_ip:$remote_log_dir/"'$log_file'" $local_log_dir/$port.log
    fi
  done
done

project=ott-client-cms
remote_log_dir=/opt/app/ruby/$project/current/log

for server in {13..15}; do
  server_ip="10.xxx.32.$server"
  local_log_dir=$LOCAL_LOG_PATH/$project/$server_ip
  log_file="nginx.access.20150815.log"

  if [[ ! -e $local_log_dir/"$log_file" ]]; then
    mkdir -p $local_log_dir
    scp $server_ip:$remote_log_dir/$log_file $local_log_dir/$log_file
  fi
done

执行:

$./copy_log 

执行后拷贝的日志如下

thin server (0.0.0.0:5200)_2015-08-15.log                                                                           100%   37MB   4.1MB/s   00:09    
thin server (0.0.0.0:5201)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5202)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5203)_2015-08-15.log                                                                           100%   37MB   3.7MB/s   00:10    
thin server (0.0.0.0:5204)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5205)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5206)_2015-08-15.log                                                                           100%   37MB   2.6MB/s   00:14    
thin server (0.0.0.0:5207)_2015-08-15.log                                                                           100%   37MB   3.7MB/s   00:10    
thin server (0.0.0.0:5200)_2015-08-15.log                                                                           100%   37MB   6.2MB/s   00:06    
thin server (0.0.0.0:5201)_2015-08-15.log                                                                           100%   37MB   4.1MB/s   00:09    
thin server (0.0.0.0:5202)_2015-08-15.log                                                                           100%   37MB   3.7MB/s   00:10    
thin server (0.0.0.0:5203)_2015-08-15.log                                                                           100%   37MB   3.1MB/s   00:12    
thin server (0.0.0.0:5204)_2015-08-15.log                                                                           100%   37MB   3.4MB/s   00:11    
thin server (0.0.0.0:5205)_2015-08-15.log                                                                           100%   37MB   3.4MB/s   00:11    
thin server (0.0.0.0:5206)_2015-08-15.log                                                                           100%   37MB   2.7MB/s   00:14    
thin server (0.0.0.0:5207)_2015-08-15.log                                                                           100%   37MB   2.7MB/s   00:14    
thin server (0.0.0.0:5200)_2015-08-15.log                                                                           100%   37MB   3.4MB/s   00:11    
thin server (0.0.0.0:5201)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5202)_2015-08-15.log                                                                           100%   37MB   4.6MB/s   00:08    
thin server (0.0.0.0:5203)_2015-08-15.log                                                                           100%   37MB   6.2MB/s   00:06    
thin server (0.0.0.0:5204)_2015-08-15.log                                                                           100%   37MB   3.1MB/s   00:12    
thin server (0.0.0.0:5205)_2015-08-15.log                                                                           100%   37MB   3.4MB/s   00:11    
thin server (0.0.0.0:5206)_2015-08-15.log                                                                           100%   37MB   4.7MB/s   00:08    
thin server (0.0.0.0:5207)_2015-08-15.log                                                                           100%   37MB   3.7MB/s   00:10    
thin server (0.0.0.0:3201)_2015-08-15.log                                                                           100%   64KB  63.7KB/s   00:00    
thin server (0.0.0.0:3202)_2015-08-15.log                                                                           100%   63KB  63.2KB/s   00:00    
thin server (0.0.0.0:3203)_2015-08-15.log                                                                           100%   69KB  68.9KB/s   00:00    
thin server (0.0.0.0:3204)_2015-08-15.log                                                                           100%   67KB  66.9KB/s   00:00    
thin server (0.0.0.0:3205)_2015-08-15.log                                                                           100%   55KB  55.5KB/s   00:00    
thin server (0.0.0.0:3206)_2015-08-15.log                                                                           100%   63KB  63.0KB/s   00:00    
thin server (0.0.0.0:3207)_2015-08-15.log                                                                           100%   57KB  56.7KB/s   00:00    
thin server (0.0.0.0:3208)_2015-08-15.log                                                                           100%   60KB  60.4KB/s   00:00    
scp: /opt/logs/m-cms-for-tudou-tv/thin server (0.0.0.0:3201)_2015-08-15.log: No such file or directory
thin server (0.0.0.0:3202)_2015-08-15.log                                                                           100%   51KB  50.7KB/s   00:00    
thin server (0.0.0.0:3203)_2015-08-15.log                                                                           100%   51KB  50.6KB/s   00:00    
thin server (0.0.0.0:3204)_2015-08-15.log                                                                           100%   52KB  52.5KB/s   00:00    
thin server (0.0.0.0:3205)_2015-08-15.log                                                                           100%   52KB  52.0KB/s   00:00    
thin server (0.0.0.0:3206)_2015-08-15.log                                                                           100%   49KB  49.1KB/s   00:00    
thin server (0.0.0.0:3207)_2015-08-15.log                                                                           100%   52KB  52.3KB/s   00:00    
thin server (0.0.0.0:3208)_2015-08-15.log                                                                           100%   52KB  52.4KB/s   00:00    
thin server (0.0.0.0:3201)_2015-08-15.log                                                                           100%   45KB  44.9KB/s   00:00    
thin server (0.0.0.0:3202)_2015-08-15.log                                                                           100%   44KB  43.7KB/s   00:00    
thin server (0.0.0.0:3203)_2015-08-15.log                                                                           100%   46KB  45.6KB/s   00:00    
thin server (0.0.0.0:3204)_2015-08-15.log                                                                           100%   46KB  46.1KB/s   00:00    
thin server (0.0.0.0:3205)_2015-08-15.log                                                                           100%   43KB  43.4KB/s   00:00    
thin server (0.0.0.0:3206)_2015-08-15.log                                                                           100%   46KB  45.7KB/s   00:00    
thin server (0.0.0.0:3207)_2015-08-15.log                                                                           100%   44KB  44.5KB/s   00:00    
thin server (0.0.0.0:3208)_2015-08-15.log                                                                           100%   44KB  43.6KB/s   00:00    
nginx.access.20150815.log                                                                                           100%   81MB   4.3MB/s   00:19    
nginx.access.20150815.log                                                                                           100%   81MB   4.1MB/s   00:20    
nginx.access.20150815.log                                                                                           100%   81MB   2.7MB/s   00:30    

 

 

(2)分析日志

分析日志的脚本如下:

#!/usr/bin/env bash

if [[ $1 == "--debug" ]]; then
  set -x
fi

LOG_PATH=~/Documents/logs/8-15
TMP_FILE=/tmp/analyse_cat.log

tv() {
  project=$1
  log_files=$LOG_PATH/$project/**/*.log

  for hour in {15..16}; do
    for min in {0..5}; do
      if [[ $hour -lt 10  ]]; then
        str="2015-08-15 0$hour:$min"
      else
        str="2015-08-15 $hour:$min"
      fi  

      request_count=$(grep "$str" $log_files | wc -l)
      response_count=$(grep "^$hour:$min" $log_files | grep Completed | wc -l)
      grep "^$hour:$min" $log_files | grep Completed > $TMP_FILE

      echo "------- $str 时总请求数: $request_count,响应数:$response_count"
      echo '响应码:'
      cut -d ' ' -f4 $TMP_FILE | sort -n | uniq -c
      # echo '响应时间:'
      # cut -d ' ' -f7 $TMP_FILE | sort -n | uniq -c | sort -nr
    done
  done
}

echo '*** Youku TV ***'
tv m-cms-for-tv

echo '*** Tubou TV ***'
tv m-cms-for-tudou-tv

执行

$ ./analyse_log

执行后的结果

~/Desktop:$ ./analyse_log 
*** Youku TV ***
------- 2015-08-15 15:0 时总请求数: 10934,响应数:10934
响应码:
  10934 200
------- 2015-08-15 15:1 时总请求数: 10908,响应数:10908
响应码:
  10908 200
------- 2015-08-15 15:2 时总请求数: 10903,响应数:10903
响应码:
  10903 200
------- 2015-08-15 15:3 时总请求数: 7184,响应数:7160
响应码:
   7160 200
------- 2015-08-15 15:4 时总请求数: 552,响应数:0
响应码:
------- 2015-08-15 15:5 时总请求数: 194063,响应数:193978
响应码:
 193978 200
------- 2015-08-15 16:0 时总请求数: 142234,响应数:142233
响应码:
 142233 200
------- 2015-08-15 16:1 时总请求数: 195244,响应数:195244
响应码:
 195244 200
------- 2015-08-15 16:2 时总请求数: 233320,响应数:233342
响应码:
 233342 200
------- 2015-08-15 16:3 时总请求数: 12787,响应数:12788
响应码:
  12788 200
------- 2015-08-15 16:4 时总请求数: 11341,响应数:11341
响应码:
  11341 200
------- 2015-08-15 16:5 时总请求数: 12470,响应数:12470
响应码:
  12470 200
*** Tubou TV ***
------- 2015-08-15 15:0 时总请求数: 5,响应数:5
响应码:
      5 200
------- 2015-08-15 15:1 时总请求数: 3,响应数:3
响应码:
      3 200
------- 2015-08-15 15:2 时总请求数: 39,响应数:39
响应码:
     39 200
------- 2015-08-15 15:3 时总请求数: 11,响应数:3
响应码:
      3 200
------- 2015-08-15 15:4 时总请求数: 31,响应数:0
响应码:
------- 2015-08-15 15:5 时总请求数: 1315,响应数:1288
响应码:
   1288 200
------- 2015-08-15 16:0 时总请求数: 9,响应数:9
响应码:
      9 200
------- 2015-08-15 16:1 时总请求数: 2,响应数:2
响应码:
      2 200
------- 2015-08-15 16:2 时总请求数: 39,响应数:39
响应码:
     39 200
------- 2015-08-15 16:3 时总请求数: 10,响应数:10
响应码:
     10 200
------- 2015-08-15 16:4 时总请求数: 2,响应数:2
响应码:
      2 200
------- 2015-08-15 16:5 时总请求数: 15,响应数:15
响应码:
     15 200

 

你可能感兴趣的:(log,日志分析,Rails)