awstats与nginx结合使用

 最近客户希望安装一个日志分析软件，于是就想到了awstats，记得以前是用awstats和apache结合使用的，但是客户使用的是nginx，这个有点麻烦，因为awstats和nginx结合的并不好，尤其是日志格式，这需要自己把nginx的日志格式转换成awstats能够识别的格式。

安装环境：

CentOS 5.4 32位 

nginx：1.0.4

awstats：7.0

一、设置好yum源

1.32位设置：

wget http://packages.sw.be/rpmforge-release/rpmforge-release-0.5.1-1.el5.rf.i386.rpm

rpm -Uvh rpmforge-release-0.5.1-1.el5.rf.i386.rpm 

2.64位设置

wget http://packages.sw.be/rpmforge-release/rpmforge-release-0.5.1-1.el5.rf.x86_64.rpm

rpm -Uvh rpmforge-release-0.5.1-1.el5.rf.x86_64.rpm

3.编辑repo文件

vim rpmforge.repo

enabled = 0

二、安装、配置awstats

yum install --enablerepo=rpmforge awstats

cp /etc/awstats/awstats.localhost.localdomain.conf

 /etc/awstats/awstats.www.example.com.conf

vim /etc/awstats/awstats.www.example.com.conf

LogFile=gzip -d

LogFormat = "%host %other %logname %time1 %methodurl %code %bytesd %refererquot %uaquot"  //指定日志格式

SiteDomain="www.example.com"  //指定域名

HostAliases="127.0.0.1 www.example.cn example.cn"

DirData="/var/www/awstats/data/www.example.com"

SkipHosts="127.0.0.1 61.129.13.29 184.73.162.74"

SkipFiles="/nginx_status /index.html"

LoadPlugin="geoip GEOIP_STANDARD /usr/share/GeoIP/GeoIP.dat" //geoip可以通过来访者的ip，定位他的经纬度，国家/地区，省市，甚至街道等位置信息。GeoIP.dat会通过下面download_geoip_db.sh脚步定时更新

三、安装geoip插件

yum install --enablerepo=rpmforge GeoIP-devel perl-Geo-IP

以下是download_geoip_db.sh的脚步

cat download_geoip_db.sh
#!/bin/bash/
#################################################
# download_geoip_db.sh
# this script will down load geoip information to $GEOIP_DES
#################################################
# ChangeLog
#
#

GEOIP_DES="/usr/share/GeoIP/"
GEOIP_DB_URL="http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
GEOIP_DB_TMP="/tmp/GeoIP.dat.gz"

REPORT_MAIL_TO="xxxx"   #这边填上你的邮件地址
REPORT_MAIL_CC="xxxx"


##################
# Error handling
#
# sum-up all the errors during the script
MAIL_ERROR_REPORT=
#ERROR=0

# add to error logs and logger
handle_error () {
    error_message=
    error_message="$1"
    logger "$(basename $0) - ERROR - $error_message"
    echo "$error_message" | mail -s "[ KEMCO - GEOIP ERROR ] an error occured during update" -c $REPORT_MAIL_CC $REPORT_MAIL_TO
    exit 1
}

# check for destination folder
if [ ! -d "$GEOIP_DES" ]; then
        mkdir -p "$GEOIP_DES"
        if [ $? -ne 0 ]; then
        handle_error "Cannot create $GEOIP_DES folder.\nOperation aborted."
    fi
fi
if [ ! -w "$GEOIP_DES" ]; then
        handle_error "Destination folder $GEOIP_DES is not writable.\nOperation aborted."
fi

#################
# Downloading geoip_db information from maxmind
#
wget $GEOIP_DB_URL -O $GEOIP_DB_TMP
if [ $? -ne 0 ]; then
    handle_error "Error while downloading GeoIP from $GEOIP_DB_URL"
fi

# check if download success
if [ ! -f "$GEOIP_DB_TMP" ]; then
    handle_error "Download GeoIP failed - temp file: $GEOIP_DB_TMP was not found"
fi

# move the geoip_db to place
gunzip -f $GEOIP_DB_TMP
if [ $? -ne 0 ]; then
    handle_error "Error while unzipping the GeoIP database"
fi

mv -f /tmp/GeoIP.dat $GEOIP_DES
if [ $? -ne 0 ]; then
    handle_error "Error while moving the GeoIP database to $GEOIP_DES"
fi

# send mail to inform this action
GEOIP_DB_TIMESTAMP=$(stat -c %y $GEOIP_DES/GeoIP.dat | cut -f1 -d' ')
echo "The update of GeoIP database is successfull.
GeoIP version: $GEOIP_DB_TIMESTAMP" | mail -s "[ KEMCO - GEOIP SUCCESS ] The GeoIP database was successfully updated on $(hostname)" -c $REPORT_MAIL_CC $REPORT_MAIL_TO

编辑cron，定时更新geoip数据库

echo '0 0 1,5 * * root bash /opt/download_geoip_db.sh' >/etc/cron.d/updateGeoIP

四、配置nginx的日志 

vim /etc/logrotate_nginx.conf

 

    daily

    rotate 3000

    missingok

    notifempty

    dateext

    compress

    sharedscripts

    postrotate

        [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`

    endscript

}

 

mkdir /etc/cron.jiuri/

vim /etc/cron.jiuri/logrotate

#!/bin/sh

/usr/sbin/logrotate /etc/logrotate_nginx.conf

EXITVALUE=$?

if [ $EXITVALUE != 0 ]; then

    /usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE] nginx"

fi

exit 0

编辑cron，每天23:59分执行nginx的logrotate

echo '59 23 * * *     root bash /etc/cron.jiuri/logrotate' >/etc/cron.d/nginxlogrotate  

 

五、更新awstats的日志目录

 当然用脚本自动去更新了，脚本如下

 

cat merge_nginx_log.sh
#!/bin/bash
# Description : This script is for the awstat to merge logs and update awstat data
# 2010-10-09
#

AWSTAT_BIN="/var/www/awstats/awstats.pl"
YEAR=$(date +%Y)
MONTH=$(date +%m)
DAY=$(date +%d)
DATE=$(date '+%Y%m%d' -d "1 day ago")
LOG_DIR=/var/log/nc_merge_nginx_log
LOCK_FILE=/var/lock/subsys/nc_merge_nginx_log
SSH_RSA_KEY=/home/ncadmin/.ssh/id_rsa

SERVER=( "srv-example-web1" "srv-example-web2" "srv-example-web3" "srv-example-web4" )
NGINX_LOG_DIR="/var/log/nginx/www.example.com"
NGINX_LOG="mixi.example.mobi_access.log"
SAVE_FOLDER="/var/www/awstats/nginx_logs/ww.example.com"
DST_FOLDER="$SAVE_FOLDER"/"$YEAR"/"$MONTH"/"$DAY"    //nginx日志转换格式后保存的目录
AWSTAT_CFG=mixi.example.mobi # IN "/etc/awstats/awstats.www.example.com.conf"


define_log(){
        LOG_PATH="$LOG_DIR"/"$YEAR"/"$MONTH"/"$DAY"
        [ ! -d "$LOG_PATH" ] && mkdir -p "$LOG_PATH"

        LOG="$LOG_DIR"/"$YEAR"/"$MONTH"/"$DAY"/nc_merge_nginx_log.log
        echo $LOG
}


get_log_from_nodes(){        //把压缩后的nginx log拷贝到awstats的目录里并解压，awstats会根据这里的log提取信息
        for srv in ${SERVER[*]}
        do
                [ ! -d "$DST_FOLDER" ] && mkdir -p "$DST_FOLDER"
                scp -i $SSH_RSA_KEY ncadmin@"$srv":"$NGINX_LOG_DIR"/"$NGINX_LOG"-"$DATE".gz "$DST_FOLDER"/"$NGINX_LOG"-"$DATE"-"$srv".log.gz
                gunzip -f "$DST_FOLDER"/"$NGINX_LOG"-"$DATE"-"$srv".log.gz
        done
}

clean_old_log(){   //清理超过3天的awstats目录里的日志，就是拷贝过来的日志
        keep_day=3
        find "$SAVE_FOLDER" -type f -ctime +$((keep_day-1)) -exec rm -rf '{}' ';'
}

update(){
        $AWSTAT_BIN -config="$AWSTAT_CFG" -configdir=/etc/awstats -update -showsteps
        $AWSTAT_BIN -config="$AWSTAT_CFG" -configdir=/etc/awstats -update -showsteps  -databasebreak=day
}

if [ -f "$LOCK_FILE" ] ;then
        echo "There is a same script running!" >> $LOG 2>&1
else
        touch "$LOCK_FILE"
        define_log
        get_log_from_nodes >> $LOG 2>&1
        update >> $LOG 2>&1
        clean_old_log >> $LOG 2>&1
        rm -rf "$LOCK_FILE"
fi

echo '59 3 * * *     root bash -x /opt/ncscripts/nmerge_nginx_log.sh' >/etc/cron.d/awstat_update

六、安装 perl-fastcgi

yum install perl-FCGI perl-pmtools

perl -MCPAN -e 'install FCGI'

 

• 
  vim /etc/init.d/perl-fastgi 
• #!/bin/bash 
• # 
• # perl-fastcgi - this script starts and stops the perl-fastcgi Daemon 
• # 
• # chkconfig:   - 85 15  
• # description:  perl-fastcgi is an perl FASTCGI Daemon 
• # processname: perl-fastcgi 
•  
• PERL_SCRIPT=/usr/bin/fastcgi-wrapper.pl 
• FASTCGI_USER=nginx 
• RETVAL=0 
• start(){ 
•       echo -n 'Perl FASTCGI Starting ...' 
•       $PERL_SCRIPT 
•       RETVAL=$? 
•       [ $RETVAL -ne 0 ] && echo -e "\t [FAIL]" || echo -e "\t [OK]" 
• } 
•  
• stop(){ 
•       echo -n 'Perl FASTCGI Stopping ...' 
•       killall  fastcgi-wrapper.pl 
•       RETVAL=$? 
•       [ $RETVAL -ne 0 ] && echo -e "\t [FAIL]" || echo -e "\t [OK]" 
• } 
•  
• case "$1" in 
•     start) 
•         start 
•   ;; 
•     stop) 
•         stop 
•   ;; 
•     restart) 
•         stop 
•         sleep 1 
•         start 
•   ;; 
•     *) 
•       echo "Usage: perl-fastcgi {start|stop|restart}" 
•       exit 1 
•   ;; 
• esac 
• exit $RETVAL 

 

mkdir /var/run/nginx

 

vim /usr/bin/fastcgi-wrapper.pl 

        $socket = FCGI::OpenSocket( "127.0.0.1:8999", 200 ); #use IP sockets

 

/etc/init.d/perl-fastcgi start

七、配置nginx

vim /etc/nginx/conf.d/awstats.conf

server {

          server_name   _;

          listen  30443;

 

          ssl on;

          ssl_certificate /etc/nginx/ssl_key/example.crt;

          ssl_certificate_key /etc/nginx/ssl_key/example.key;

          ssl_session_timeout 5m;

          ssl_protocols SSLv2 SSLv3 TLSv1;

          ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;

          ssl_prefer_server_ciphers on;

 

          auth_basic            "Restricted";

          auth_basic_user_file  conf.d/.example_auth;

 

          access_log   /var/log/nginx/awstats/access_logs.example.com.log  main;

          error_log   /var/log/nginx/awstats/error_logs.example.com.log;

 

          root /var/www/awstats/;

          index index.html;

 

          location ~* .*\.pl$ {

               root /var/www/awstats/;

               fastcgi_pass 127.0.0.1:8999;

               fastcgi_index awstats.pl;

               #fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;

               fastcgi_param SCRIPT_FILENAME  /var/www/awstats/awstats.pl;

               fastcgi_param QUERY_STRING     $query_string;

               fastcgi_param REQUEST_METHOD   $request_method;

               fastcgi_param CONTENT_TYPE     $content_type;

               fastcgi_param CONTENT_LENGTH   $content_length;

               fastcgi_param GATEWAY_INTERFACE CGI/1.1;

               fastcgi_param SERVER_SOFTWARE    nginx;

               fastcgi_param SCRIPT_NAME        $fastcgi_script_name;

               fastcgi_param REQUEST_URI        $request_uri;

               fastcgi_param DOCUMENT_URI       $document_uri;

               fastcgi_param DOCUMENT_ROOT      $document_root;

               fastcgi_param SERVER_PROTOCOL    $server_protocol;

               fastcgi_param REMOTE_ADDR        $remote_addr;

               fastcgi_param REMOTE_PORT        $remote_port;

               fastcgi_param SERVER_ADDR        $server_addr;

               fastcgi_param SERVER_PORT        $server_port;

               fastcgi_param SERVER_NAME        $server_name;

           }

 

          location ~ ^/awstats/ {

                root /var/www/awstats/;

                index index.html;

 

         }

}

 

/etc/init.d/nginx reload

 

八、测试

https://www.example:30443/awstats.pl?config=www.example.com