最近客户希望安装一个日志分析软件,于是就想到了awstats,记得以前是用awstats和apache结合使用的,但是客户使用的是nginx,这个有点麻烦,因为awstats和nginx结合的并不好,尤其是日志格式,这需要自己把nginx的日志格式转换成awstats能够识别的格式。
安装环境:
CentOS 5.4 32位
nginx:1.0.4
awstats:7.0
一、设置好yum源
1.32位设置:
wget http://packages.sw.be/rpmforge-release/rpmforge-release-0.5.1-1.el5.rf.i386.rpm
rpm -Uvh rpmforge-release-0.5.1-1.el5.rf.i386.rpm
2.64位设置
wget http://packages.sw.be/rpmforge-release/rpmforge-release-0.5.1-1.el5.rf.x86_64.rpm
rpm -Uvh rpmforge-release-0.5.1-1.el5.rf.x86_64.rpm
3.编辑repo文件
vim rpmforge.repo
enabled = 0
二、安装、配置awstats
yum install --enablerepo=rpmforge awstats
cp /etc/awstats/awstats.localhost.localdomain.conf
/etc/awstats/awstats.www.example.com.conf
vim /etc/awstats/awstats.www.example.com.conf
LogFile=gzip -d </var/log/nginx/access.log.gz | //指定日志位置
LogFormat = "%host %other %logname %time1 %methodurl %code %bytesd %refererquot %uaquot" //指定日志格式
SiteDomain="www.example.com" //指定域名
HostAliases="127.0.0.1 www.example.cn example.cn"
DirData="/var/www/awstats/data/www.example.com"
SkipHosts="127.0.0.1 61.129.13.29 184.73.162.74"
SkipFiles="/nginx_status /index.html"
LoadPlugin="geoip GEOIP_STANDARD /usr/share/GeoIP/GeoIP.dat" //geoip可以通过来访者的ip,定位他的经纬度,国家/地区,省市,甚至街道等位置信息。GeoIP.dat会通过下面download_geoip_db.sh脚步定时更新
三、安装geoip插件
yum install --enablerepo=rpmforge GeoIP-devel perl-Geo-IP
以下是download_geoip_db.sh的脚步
- cat download_geoip_db.sh
- #!/bin/bash/
- #################################################
- # download_geoip_db.sh
- # this script will down load geoip information to $GEOIP_DES
- #################################################
- # ChangeLog
- #
- #
- GEOIP_DES="/usr/share/GeoIP/"
- GEOIP_DB_URL="http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
- GEOIP_DB_TMP="/tmp/GeoIP.dat.gz"
- REPORT_MAIL_TO="xxxx" #这边填上你的邮件地址
- REPORT_MAIL_CC="xxxx"
- ##################
- # Error handling
- #
- # sum-up all the errors during the script
- MAIL_ERROR_REPORT=
- #ERROR=0
- # add to error logs and logger
- handle_error () {
- error_message=
- error_message="$1"
- logger "$(basename $0) - ERROR - $error_message"
- echo "$error_message" | mail -s "[ KEMCO - GEOIP ERROR ] an error occured during update" -c $REPORT_MAIL_CC $REPORT_MAIL_TO
- exit 1
- }
- # check for destination folder
- if [ ! -d "$GEOIP_DES" ]; then
- mkdir -p "$GEOIP_DES"
- if [ $? -ne 0 ]; then
- handle_error "Cannot create $GEOIP_DES folder.\nOperation aborted."
- fi
- fi
- if [ ! -w "$GEOIP_DES" ]; then
- handle_error "Destination folder $GEOIP_DES is not writable.\nOperation aborted."
- fi
- #################
- # Downloading geoip_db information from maxmind
- #
- wget $GEOIP_DB_URL -O $GEOIP_DB_TMP
- if [ $? -ne 0 ]; then
- handle_error "Error while downloading GeoIP from $GEOIP_DB_URL"
- fi
- # check if download success
- if [ ! -f "$GEOIP_DB_TMP" ]; then
- handle_error "Download GeoIP failed - temp file: $GEOIP_DB_TMP was not found"
- fi
- # move the geoip_db to place
- gunzip -f $GEOIP_DB_TMP
- if [ $? -ne 0 ]; then
- handle_error "Error while unzipping the GeoIP database"
- fi
- mv -f /tmp/GeoIP.dat $GEOIP_DES
- if [ $? -ne 0 ]; then
- handle_error "Error while moving the GeoIP database to $GEOIP_DES"
- fi
- # send mail to inform this action
- GEOIP_DB_TIMESTAMP=$(stat -c %y $GEOIP_DES/GeoIP.dat | cut -f1 -d' ')
- echo "The update of GeoIP database is successfull.
- GeoIP version: $GEOIP_DB_TIMESTAMP" | mail -s "[ KEMCO - GEOIP SUCCESS ] The GeoIP database was successfully updated on $(hostname)" -c $REPORT_MAIL_CC $REPORT_MAIL_TO
编辑cron,定时更新geoip数据库
echo '0 0 1,5 * * root bash /opt/download_geoip_db.sh' >/etc/cron.d/updateGeoIP
四、配置nginx的日志
vim /etc/logrotate_nginx.conf
daily
rotate 3000
missingok
notifempty
dateext
compress
sharedscripts
postrotate
[ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
endscript
}
mkdir /etc/cron.jiuri/
vim /etc/cron.jiuri/logrotate
#!/bin/sh
/usr/sbin/logrotate /etc/logrotate_nginx.conf
EXITVALUE=$?
if [ $EXITVALUE != 0 ]; then
/usr/bin/logger -t logrotate "ALERT exited abnormally with [$EXITVALUE] nginx"
fi
exit 0
编辑cron,每天23:59分执行nginx的logrotate
echo '59 23 * * * root bash /etc/cron.jiuri/logrotate' >/etc/cron.d/nginxlogrotate
五、更新awstats的日志目录
当然用脚本自动去更新了,脚本如下
- cat merge_nginx_log.sh
- #!/bin/bash
- # Description : This script is for the awstat to merge logs and update awstat data
- # 2010-10-09
- #
- AWSTAT_BIN="/var/www/awstats/awstats.pl"
- YEAR=$(date +%Y)
- MONTH=$(date +%m)
- DAY=$(date +%d)
- DATE=$(date '+%Y%m%d' -d "1 day ago")
- LOG_DIR=/var/log/nc_merge_nginx_log
- LOCK_FILE=/var/lock/subsys/nc_merge_nginx_log
- SSH_RSA_KEY=/home/ncadmin/.ssh/id_rsa
- SERVER=( "srv-example-web1" "srv-example-web2" "srv-example-web3" "srv-example-web4" )
- NGINX_LOG_DIR="/var/log/nginx/www.example.com"
- NGINX_LOG="mixi.example.mobi_access.log"
- SAVE_FOLDER="/var/www/awstats/nginx_logs/ww.example.com"
- DST_FOLDER="$SAVE_FOLDER"/"$YEAR"/"$MONTH"/"$DAY" //nginx日志转换格式后保存的目录
- AWSTAT_CFG=mixi.example.mobi # IN "/etc/awstats/awstats.www.example.com.conf"
- define_log(){
- LOG_PATH="$LOG_DIR"/"$YEAR"/"$MONTH"/"$DAY"
- [ ! -d "$LOG_PATH" ] && mkdir -p "$LOG_PATH"
- LOG="$LOG_DIR"/"$YEAR"/"$MONTH"/"$DAY"/nc_merge_nginx_log.log
- echo $LOG
- }
- get_log_from_nodes(){ //把压缩后的nginx log拷贝到awstats的目录里并解压,awstats会根据这里的log提取信息
- for srv in ${SERVER[*]}
- do
- [ ! -d "$DST_FOLDER" ] && mkdir -p "$DST_FOLDER"
- scp -i $SSH_RSA_KEY ncadmin@"$srv":"$NGINX_LOG_DIR"/"$NGINX_LOG"-"$DATE".gz "$DST_FOLDER"/"$NGINX_LOG"-"$DATE"-"$srv".log.gz
- gunzip -f "$DST_FOLDER"/"$NGINX_LOG"-"$DATE"-"$srv".log.gz
- done
- }
- clean_old_log(){ //清理超过3天的awstats目录里的日志,就是拷贝过来的日志
- keep_day=3
- find "$SAVE_FOLDER" -type f -ctime +$((keep_day-1)) -exec rm -rf '{}' ';'
- }
- update(){
- $AWSTAT_BIN -config="$AWSTAT_CFG" -configdir=/etc/awstats -update -showsteps
- $AWSTAT_BIN -config="$AWSTAT_CFG" -configdir=/etc/awstats -update -showsteps -databasebreak=day
- }
- if [ -f "$LOCK_FILE" ] ;then
- echo "There is a same script running!" >> $LOG 2>&1
- else
- touch "$LOCK_FILE"
- define_log
- get_log_from_nodes >> $LOG 2>&1
- update >> $LOG 2>&1
- clean_old_log >> $LOG 2>&1
- rm -rf "$LOCK_FILE"
- fi
echo '59 3 * * * root bash -x /opt/ncscripts/nmerge_nginx_log.sh' >/etc/cron.d/awstat_update
六、安装 perl-fastcgi
yum install perl-FCGI perl-pmtools
perl -MCPAN -e 'install FCGI'
mkdir /var/run/nginx
vim /usr/bin/fastcgi-wrapper.pl
$socket = FCGI::OpenSocket( "127.0.0.1:8999", 200 ); #use IP sockets
/etc/init.d/perl-fastcgi start