syslog-ng日志服务器搭建

日志收集机器安装的就是syslog-ng，下面讲解一下syslog-ng的安装和配置步骤：
日志下载：
安装顺序：
1.eventlog--eventlog_0.2.12
地址：
https://my.balabit.com/downloads/eventlog/0.2/eventlog_0.2.12.tar.gz

2.libol---libol-0.3.15
地址：
https://my.balabit.com/downloads/libol/0.3/libol-0.3.15.tar.gz

3.syslog-ng--syslog-ng_3.3.5
地址：
https://my.balabit.com/downloads/syslog-ng/sources/3.3.5/source/syslog-ng_3.3.5.tar.gz

安装步骤：
1，安装eventlog
# tar -zxvf eventlog_0.2.12.tar.gz
# cd eventlog-0.2.12/
# ./configure   --prefix=/usr/local/eventlog && make && make install

# ls /usr/local/eventlog/ 
include   lib 

2.安装libol
# tar -zxvf libol-0.3.15.tar.gz
# cd libol-0.3.15/
# ./configure --prefix=/usr/local/libol

# ls /usr/local/libol/
bin  include  lib

3.安装syslog-ng
# tar -zxvf syslog-ng_3.3.5.tar.gz
# cd syslog-ng-3.3.5/

# export PKG_CONFIG_PATH=/usr/local/eventlog/lib/pkgconfig

# ./configure --prefix=/usr/local/syslog-ng --with-libol=/usr/local/libol

ls /usr/local/syslog-ng/ 
bin  etc  include  lib  libexec  sbin  share  var

cp ./contrib/syslog-ng.conf.RedHat /usr/local/syslog-ng/etc/  #---拷贝一个配置案例作为参考

#########配置自启动##########
# cp ./contrib/init.d.RedHat /etc/init.d/syslog-ng          #----拷贝自启动案例文件
# chmod +x /etc/init.d/syslog-ng
# chkconfig --add syslog-ng
service syslog-ng does not support chkconfig  #--------这个提示，请修改自启动文本

#/etc/init.d/syslog-ng--脚本头部增加以下几句代码作为声明
#!/bin/bash 
#chkconifg: --add syslog-ng 
#chkconfig: 2345 12 88 
#Description: syslog-ng

####----该脚本还需要修改下面的三个位置
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/syslog-ng/bin:/usr/local/syslog-ng/sbin

INIT_PROG="/usr/local/syslog-ng/sbin/syslog-ng"     # 服务运行地址
INIT_OPTS="-f /usr/local/syslog-ng/etc/syslog-ng.conf"     # 服务配置文件地址

###########自启动配置完成#########

###########日志服务器配置文件#####
#/usr/local/syslog-ng/etc/syslog-ng.conf ---配置文件修改内容为
#如下；

service:
#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#

@version : 3.3
@include "scl.conf"

options {
    flush_lines (0);
    time_reopen (2);
    log_fifo_size (50000);
    chain_hostnames (no);
    use_dns (no);
    use_fqdn (no);
    keep_hostname (no);
    perm (0755);
    dir_perm (0755);
    create_dirs (yes);
};

source s_sys {
    file ("/proc/kmsg" program_override("kernel: "));
    unix-stream ("/dev/log" max-connections(500) log_fetch_limit(20) log_iw_size(50000));
};

# destinations
destination d_messages { file("/var/log/messages"); };
destination d_cons { file("/dev/console"); };
destination d_mesg { file("/var/log/messages"); };
destination d_auth { file("/var/log/secure"); };
destination d_mail { file("/var/log/maillog" flush_lines(10)); };
destination d_spol { file("/var/log/spooler"); };
destination d_boot { file("/var/log/boot.log"); };
destination d_cron { file("/var/log/cron"); };
destination d_mlal { usertty("*"); };

# filter
filter f_filter2   { level(info..emerg) and
                     not facility(mail,authpriv,cron,local6,local7); };
filter f_filter3   { facility(authpriv); };
filter f_filter4   { facility(mail); };
filter f_filter5   { level(emerg); };
filter f_filter6   { facility(uucp) or
                     (facility(news) and level(crit..emerg)); };
filter f_filter7   { facility(local7); };
filter f_filter8   { facility(cron); };

# log
log { source(s_sys); filter(f_filter2); destination(d_mesg); };
log { source(s_sys); filter(f_filter3); destination(d_auth); };
log { source(s_sys); filter(f_filter4); destination(d_mail); };
log { source(s_sys); filter(f_filter5); destination(d_mlal); };
log { source(s_sys); filter(f_filter6); destination(d_spol); };
log { source(s_sys); filter(f_filter7); destination(d_boot); };
log { source(s_sys); filter(f_filter8); destination(d_cron); };

# for syslog-ng
source s_syslogng {internal();};
destination d_syslogng {file ("/memp/logs/syslog-ng/$YEAR/$MONTH-$DAY/syslog-ng-$HOUR.log" create_dirs(yes) perm (0640) dir_perm (0750));};
log { source(s_syslogng); destination(d_syslogng);};

#for php_error.log
source s_phperror { tcp(ip(192.168.1.102), port(1999), so_keepalive(yes)); };
filter f_phperro { host("192.168.1.250,192.168.1.102") };
destination d_phperror { file("/memp/logs/phperror/$HOST/$YEAR-$MONTH-$DAY/php_error.log" create_dirs(yes) template("[$HOST] $DATE $PROGRAM $MESSAGE\n"));};

source s_nginx { tcp(ip(192.168.1.102), port(2999), so_keepalive(yes)); };
filter f_nginx { host("192.168.1.102") or host("192.168.1.250")};
destination d_nginx { file("/memp/logs/nginx_access/$HOST/$YEAR-$MONTH-$DAY/nginx-access.log" create_dirs(yes) template("[$HOST] $DATE $PROGRAM $MESSAGE\n"));};

log{ source(s_phperror); filter(f_phperro); destination(d_phperror);};
log{ source(s_nginx); filter(f_nginx); destination(d_nginx);};

#for user behavior
source s_behavior{
    syslog(ip(0.0.0.0)
    port(6999)
    transport("tcp")
    max-connections(1000)
    log_fetch_limit(80000)
    log_iw_size(800000)
    );
};

filter f_behavior{level(info);};
filter f_behavior_local6{facility(local6);};

parser p_behavior{
    csv-parser(
    columns("ACTION",'DETAIL')
    delimiters("|")
    flags(drop-invalid, escape-none, greedy)
    template("${MESSAGE}")
);
};

destination d_behavior {file("/memp/logs/$PROGRAM/$YEAR-$MONTH-$DAY/$ACTION-$HOUR.log");};

log{
    source(s_behavior);
    filter(f_behavior_local6);filter(f_behavior);
    parser(p_behavior);
    destination(d_behavior);
    flags(flow-control);
};

-------------------------------------
client:

#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#

@version : 3.3
@include "scl.conf"

options {
    flush_lines (0);
    time_reopen (2);
    log_fifo_size (50000);
    chain_hostnames (no);
    use_dns (no);
    use_fqdn (no);
    keep_hostname (yes);
    perm (0755);
    dir_perm (0755);
    create_dirs (yes);
};

source s_sys {
    file ("/proc/kmsg" program_override("kernel: "));
    unix-stream ("/dev/log" max-connections(500) log_fetch_limit(20) log_iw_size(50000));
};

# destinations
destination d_messages { file("/var/log/messages"); };
destination d_cons { file("/dev/console"); };
destination d_mesg { file("/var/log/messages"); };
destination d_auth { file("/var/log/secure"); };
destination d_mail { file("/var/log/maillog" flush_lines(10)); };
destination d_spol { file("/var/log/spooler"); };
destination d_boot { file("/var/log/boot.log"); };
destination d_cron { file("/var/log/cron"); };
destination d_mlal { usertty("*"); };

# filter
filter f_filter2   { level(info..emerg) and
                     not facility(mail,authpriv,cron,local6,local7); };
filter f_filter3   { facility(authpriv); };
filter f_filter4   { facility(mail); };
filter f_filter5   { level(emerg); };
filter f_filter6   { facility(uucp) or
                     (facility(news) and level(crit..emerg)); };
filter f_filter7   { facility(local7); };
filter f_filter8   { facility(cron); };

# log
log { source(s_sys); filter(f_filter2); destination(d_mesg); };
log { source(s_sys); filter(f_filter3); destination(d_auth); };
log { source(s_sys); filter(f_filter4); destination(d_mail); };
log { source(s_sys); filter(f_filter5); destination(d_mlal); };
log { source(s_sys); filter(f_filter6); destination(d_spol); };
log { source(s_sys); filter(f_filter7); destination(d_boot); };
log { source(s_sys); filter(f_filter8); destination(d_cron); };

# for syslog-ng
source s_syslogng {internal();};
destination d_syslogng {file ("/letv/logs/syslog/$YEAR/$MONTH/$DAY/syslog-ng.log" create_dirs(yes) perm (0640) dir_perm (0750));};
log { source(s_syslogng); destination(d_syslogng);};

# for php_error.log
source s_phperror { file("/home/wwwroot/www.memp.com/api/protected/runtime/application.log" follow_freq(1) flags(no-parse)); };
destination d_phperror {tcp( "192.168.1.102" port(1999));};
log { source(s_phperror);destination(d_phperror);};
################################################################
source s_nginx { file("/home/wwwlogs/www.marryme.com/access.log" follow_freq(1) flags(no-parse)); };
destination d_nginx {tcp( "192.168.1.102" port(2999));};
log { source(s_nginx);destination(d_nginx);};

# for user behavior log

filter f_behavior_local6{ facility(local6); };
destination d_logremote { syslog("192.168.1.102" transport("tcp") port(6999) keep-alive(yes) log_fifo_size(300000) );};
log { source(s_sys); filter(f_behavior_local6);  destination(d_logremote); flags(flow-control);};

##关闭旧syslog日志系统
service rsyslog stop
chkconfig --list rsyslog
chkconfig rsyslog off

--开启
chkconfig --level 2345 rsyslog on
#########

下面不用看
#################开启防火墙时-开发端口################
iptables -A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 514 -j ACCEPT
iptables -A INPUT -p udp -m udp -s 192.168.0.0/16 --dport 514 -j ACCEPT
######################################################

#####perl解析模块#####
perl -e 'use Text::CSV' #检测是否安装
perl -MCPAN -e shell
yum install perl-CPAN
perl -MCPAN -e shell
install Text::CSV