Nginx 和 syslog-ng 实现日志收集

1、10.105.20.11 web服务
cat /etc/nginx/nginx.conf

日志格式配置：

$http_x_forwarded_for 可以获取IP
log_format main '$http_x_forwarded_for   $time_local     $request        $ddshowcookie   $status $http_user_agent        $http_x_forwarded_for      $request_time   $connection     $http_referer';

cat /etc/nginx/conf.d/ddshow.conf

location ~ ^/tj {
  add_header Content-Type "text/plain;charset=utf-8";
 # return 200 "static ok , Your IP Address:$http_x_forwarded_for";
  proxy_pass http://xiu_youku_com;
  access_log /opt/logs/nginx/access/tj_access_file.log.pipe main;
  }

 
所有星梦的请求会再访问一次 http://www.xingmeng.com/tj 留下访问日志。

access_log /opt/logs/nginx/access/tj_access_file.log.pipe(产生的文件) main(日志格式)

2、cat /etc/syslog-ng.conf

source s_nginx_access { pipe ("/opt/logs/nginx/access/tj_access_file.log.pipe" );}
模板：
template t_filetemplate { template("$MSG\n"); template_escape(no); };
目的：
destination d_log_access { udp("10.105.20.111" port(1234) );  };  upd方式
destination d_local_access { file("/opt/logs/nginx/access/tj_access_file.log" template(t_filetemplate));};  本地文件方式
过程：
log { source(s_nginx_access); destination(d_local_access);destination(d_log_access); };

源：


3、10.105.20.111
cat /etc/syslog-ng.conf

源：
source s_nginx_ddshow { udp(ip(0.0.0.0) port(1234)); };

目的：
destination d_nginx_ddshow {
    file ("/opt/data/syslog/ddshow_$YEAR-$MONTH-$DAY.log" ); 
};

过程：
log { source(s_nginx_ddshow);  destination(d_nginx_ddshow); };

4、10.105.20.101
crontab -l

# copy log
30 0 * * *  sh /opt/cafe/shell/log_cp.sh
#Transfer log from from 10105.20.111 to 10.105.20.101
30 0 * * * rsync -avzp 10.105.20.111::cafe_log/ddshow_`date --date="1 days ago"  +\%Y-\%m-\%d`.log /opt/data/syslog/ws/

其中 cat /opt/cafe/shell/log_cp.sh

#!/bin/bash
#这个脚本必须在每天的一点10 运行

day_1=$(date -d "yesterday" +"%Y-%m-%d")
mkdir -p /opt/data/syslog/ws
scp -P 22022 [email protected]:/opt/data/syslog/ddshow_${day_1}.log /opt/data/syslog/ws/ddshow_${day_1}.log

10.105.20.101 的rsync配置如下：

# sample rsyncd.conf configuration file

# GLOBAL OPTIONS

#motd file=/etc/motd
#log file=/var/log/rsyncd
# for pid file, do not use /var/run/rsync.pid if
# you are going to run rsync out of the init.d script.
# pid file=/var/run/rsyncd.pid
#syslog facility=daemon
#socket options=

# MODULE OPTIONS

[cafe_log]
path = /opt/data/syslog
comment = cafe
ignore errors
read only = yes
list = false
uid=cafe
gid=cafe
#auth users = backup
#secrets file = /etc/rsyncd.secrets

其中 cafe_log = /opt/data/syslog