2019独角兽企业重金招聘Python工程师标准>>> 
最终主机需求
192.168.40.83
iptables2 logstash
192.168.40.103
test2 filebeat
192.168.40.105
test5 elasticsearch kibana
软件包为：
jdk-8u101-linux-x64.rpm
logstash-2.3.2.tar.gz
filebeat-1.2.3-x86_64.rpm
elasticsearch-2.3.4.rpm
kibana-4.5.3-linux-x64.tar.gz
下载链接：
http://pan.baidu.com/s/1pLGzoYR
logstash使用篇
1.只用logstash
使用input段中file插件；从文件中获取输入
使用output段中stdout插件；输出到标准输出中
logstash安装就是直接解压即可
iptables2
ver1.conf
input {
file {
type => "ssh.login"
path => ["/var/log/secure"]
}
}
output {
stdout {}
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver1.conf
Settings: Default pipeline workers: 2
Pipeline main started
2017-04-17T02:01:42.582Z iptables2 Apr 17 10:01:41 iptables2 sshd[48946]: Accepted password for root from 192.168.40.26 port 65319 ssh2
2017-04-17T02:01:42.584Z iptables2 Apr 17 10:01:41 iptables2 sshd[48946]: pam_unix(sshd:session): session opened for user root by (uid=0)
2017-04-17T02:02:08.632Z iptables2 Apr 17 10:02:08 iptables2 sshd[48946]: Received disconnect from 192.168.40.26: 0:
2017-04-17T02:02:08.633Z iptables2 Apr 17 10:02:08 iptables2 sshd[48946]: pam_unix(sshd:session): session closed for user root
这里新开一个ssh回话和关闭一个ssh回话，/var/log/secure中都会产生新日志
[root@iptables2 ~]# cat ver1.conf
input {
file {
type => "ssh.login"
path => ["/var/log/secure"]
}
}
output {
stdout {
codec => rubydebug
}
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver1.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "Apr 17 10:49:56 iptables2 sshd[49662]: Received disconnect from 192.168.40.26: 0: ",
"@version" => "1",
"@timestamp" => "2017-04-17T02:49:57.574Z",
"path" => "/var/log/secure",
"host" => "iptables2",
"type" => "ssh.login"
}
{
"message" => "Apr 17 10:49:56 iptables2 sshd[49662]: pam_unix(sshd:session): session closed for user root",
"@version" => "1",
"@timestamp" => "2017-04-17T02:49:57.578Z",
"path" => "/var/log/secure",
"host" => "iptables2",
"type" => "ssh.login"
}
2.只用logstash
使用input段中的file插件；
使用filter段中的grok插件和date插件及条件判断语句；
使用output段中的stdout插件；
iptables2
ver2.conf
[root@iptables2 ~]# cat ver2.conf
input {
file {
type => "syslog"
path => ["/var/log/secure"]
}
}
filter {
if [type] == "syslog" {
grok {
match => [ "message", "%{SYSLOGLINE}" ]
overwrite => [ "message" ]
}
}
date {
match => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
}
}
output {
stdout {
codec => rubydebug
}
}
其中大写的SYSLOGLINE是在
https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns下定义的，也可以自己写，不过这里已经有很多可以选择了httpd和java、linux-syslog都有
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver2.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "Received disconnect from 192.168.40.26: 0: ",
"@version" => "1",
"@timestamp" => "2017-04-17T02:56:08.000Z",
"path" => "/var/log/secure",
"host" => "iptables2",
"type" => "syslog",
"timestamp" => "Apr 17 10:56:08",
"logsource" => "iptables2",
"program" => "sshd",
"pid" => "49843"
}
{
"message" => "pam_unix(sshd:session): session closed for user root",
"@version" => "1",
"@timestamp" => "2017-04-17T02:56:08.000Z",
"path" => "/var/log/secure",
"host" => "iptables2",
"type" => "syslog",
"timestamp" => "Apr 17 10:56:08",
"logsource" => "iptables2",
"program" => "sshd",
"pid" => "49843"
}
3.只用logstash
使用input段中的file插件；
使用filter段中的grok插件和date插件及if条件语句
使用output段中的file插件
[root@iptables2 ~] # cat ver3.conf
input {
file {
type => "syslog"
path => ["/var/log/secure"]
}
}
filter {
if [type] == "syslog" {
grok {
match => [ "message", "%{SYSLOGLINE}" ]
overwrite => [ "message" ]
}
}
date {
match => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
}
}
output {
# stdout {
# codec => rubydebug
# }
file {
path => "/tmp/hello.log"
}
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver3.conf
Settings: Default pipeline workers: 2
Pipeline main started
[root@iptables2 ~]# cat /tmp/hello.log
{"message":"Accepted password for root from 192.168.40.26 port 52274 ssh2","@version":"1","@timestamp":"2017-04-17T03:11:37.000Z","path":"/var/log/secure","host":"iptables2","type":"syslog","timestamp":"Apr 17 11:11:37","logsource":"iptables2","program":"sshd","pid":"50045"}
{"message":"pam_unix(sshd:session): session opened for user root by (uid=0)","@version":"1","@timestamp":"2017-04-17T03:11:37.000Z","path":"/var/log/secure","host":"iptables2","type":"syslog","timestamp":"Apr 17 11:11:37","logsource":"iptables2","program":"sshd","pid":"50045"}
{"message":"Received disconnect from 192.168.40.26: 0: ","@version":"1","@timestamp":"2017-04-17T03:12:13.000Z","path":"/var/log/secure","host":"iptables2","type":"syslog","timestamp":"Apr 17 11:12:13","logsource":"iptables2","program":"sshd","pid":"50045"}
{"message":"pam_unix(sshd:session): session closed for user root","@version":"1","@timestamp":"2017-04-17T03:12:13.000Z","path":"/var/log/secure","host":"iptables2","type":"syslog","timestamp":"Apr 17 11:12:13","logsource":"iptables2","program":"sshd","pid":"50045"}