elasticsearch
[root@server1 elk]# rpm -ivh elasticsearch-2.3.3.rpm
[root@server1 elk]# cd /etc/elasticsearch/
[root@server1 elasticsearch]# vim elasticsearch.yml
cluster.name: my-application
node.name: server1
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.1
http.port: 9200
[root@server1 ~]# rpm -ivh jdk-8u121-linux-x64.rpm
[root@server1 ~]# /etc/init.d/elasticsearch start
Starting elasticsearch: [ OK ]
[root@server1 ~]# cat /var/log/elasticsearch/my-application.log
[2018-10-26 09:47:23,233][WARN ][bootstrap ] These can be adjusted by modifying /etc/security/limits.conf, for example:
# allow user 'elasticsearch' mlockall
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server1 ~]# vim /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server1 ~]# /etc/init.d/elasticsearch restart
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
[root@server1 elk]# cd /usr/share/elasticsearch/
[root@server1 elasticsearch]# cd bin/
[root@server1 bin]# ./plugin install file:/root/elk/elasticsearch-head-master.zip
-> Installing from file:/root/elk/elasticsearch-head-master.zip…
Trying file:/root/elk/elasticsearch-head-master.zip ...
Downloading .........DONE
Verifying file:/root/elk/elasticsearch-head-master.zip checksums if available ...
NOTE: Unable to verify checksum for downloaded plugin (unable to find .sha1 or .md5 file to verify)
Installed head into /usr/share/elasticsearch/plugins/head
[root@server1 bin]# cd …
[root@server1 elasticsearch]# cd plugins/
[root@server1 plugins]# cd head/
[root@server1 head]# ls
elasticsearch-head.sublime-project index.html plugin-descriptor.properties src
Gruntfile.js LICENCE README.textile test
grunt_fileSets.js package.json _site
Server1:
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml
discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]
[root@server1 ~]# /etc/init.d/elasticsearch restart
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
Server2:
[root@server2 ~]# yum install -y elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# vim /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml
cluster.name: my-application
node.name: server2
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.2
http.port: 9200
discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]
[root@server2 ~]# /etc/init.d/elasticsearch start
Starting elasticsearch: [ OK ]
Server3:
[root@server3 ~]# yum install -y elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# vim /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml
cluster.name: my-application
node.name: server1
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.3
http.port: 9200
discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]
[root@server3 ~]# /etc/init.d/elasticsearch start
Starting elasticsearch: [ OK ]
Server1:
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml
node.data: false
node.master: true
[root@server1 ~]# /etc/init.d/elasticsearch restart
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
Server2:
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml
node.master: false
node.data: true
[root@server2 ~]# /etc/init.d/elasticsearch restart
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
Server3:
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml
node.master: false
node.data: true
[root@server3 ~]# /etc/init.d/elasticsearch restart
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
Server1:
[root@server1 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm
Preparing... ########################################### [100%]
1:logstash ########################################### [100%]
[root@server1 etc]# cd /opt/logstash/
[root@server1 logstash]# bin/logstash -e ‘input { stdin { } } output { stdout {} }’
Settings: Default pipeline workers: 1
Pipeline main started
hello world
2018-10-26T03:14:49.110Z server1 hello world
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 logstash]# bin/logstash -e ‘input { stdin { } } output { stdout {codec => rubydebug } elasticsearch { hosts => [“172.25.20.1”] index => “logstash-%{+YYYY.MM.dd}”} }’
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-10-26T03:19:43.566Z",
"host" => "server1"
}
worls
{
"message" => "worls",
"@version" => "1",
"@timestamp" => "2018-10-26T03:19:49.622Z",
"host" => "server1"
}
[root@server1 logstash]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# vim es.conf
input {
stdin {}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.20.1"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-10-26T03:33:26.958Z",
"host" => "server1"
}
world
{
"message" => "world",
"@version" => "1",
"@timestamp" => "2018-10-26T03:33:31.148Z",
"host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 conf.d]# vim es.conf
input {
stdin {}
}
output {
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.20.1"]
index => "logstash-%{+YYYY.MM.dd}"
}
file {
path => "/tmp/testfile"
codec => line { format => "custom format: %{message}"}
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
hello
world
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 conf.d]# cat /tmp/testfile
custom format: hello
custom format: world
[root@server1 conf.d]# vim es.conf
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
output {
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.20.1"]
index => "messages-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
Server2:
[root@server2 ~]# vim /etc/rsyslog.conf
# Provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
# Provides TCP syslog reception
$ModLoad imtcp
$InputTCPServerRun 514
*.* @@172.25.20.1:514
[root@server2 ~]# /etc/init.d/rsyslog restart
Shutting down system logger: [ OK ]
Starting system logger: [ OK ]
Server1:
[root@server1 conf.d]# vim es.conf
input {
# file {
# path => "/var/log/messages"
# start_position => "beginning"
# }
syslog {
port => 514
}
}
output {
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.20.1"]
index => "syslog-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
[root@server1 conf.d]# vim es.conf
input {
# file {
# path => "/var/log/elasticsearch/my-application.log"
# start_position => "beginning"
# }
# syslog {
# port => 514
# }
stdin {
codec => multiline {
pattern => "^\["
negate => "true"
what => "previous"
}
}
}
#filter {
#multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
#}
output {
stdout {
codec => rubydebug
}
# elasticsearch {
# hosts => ["172.25.20.1"]
# index => "es-%{+YYYY.MM.dd}"
# }
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
hello
hell
[
{
"@timestamp" => "2018-10-26T06:16:12.501Z",
"message" => "hello\nhell",
"@version" => "1",
"tags" => [
[0] "multiline"
],
"host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 conf.d]# vim es.conf
input {
file {
path => "/var/log/elasticsearch/my-application.log"
start_position => "beginning"
}
# syslog {
# port => 514
# }
# stdin {
# codec => multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
# }
}
filter {
multiline {
pattern => "^\["
negate => "true"
what => "previous"
}
}
output {
# stdout {
# codec => rubydebug
# }
elasticsearch {
hosts => ["172.25.20.1"]
index => "es-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 ~]# l.
. .bash_logout .cshrc .ssh
.. .bash_profile .oracle_jre_usage .tcshrc
.bash_history .bashrc .sincedb_4d10e6fb4e3ffdf0fe00d1ce0f25b0fd .viminfo
[root@server1 ~]# rm -rf .sincedb_4d10e6fb4e3ffdf0fe00d1ce0f25b0fd
[root@server1 conf.d]# yum install httpd -y
[root@server1 conf.d]# /etc/init.d/httpd start
Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.20.1 for ServerName
[ OK ]
[root@server1 conf.d]# vim es.conf
input {
file {
path => "/var/log/httpd/access_log"
start_position => "beginning"
}
# syslog {
# port => 514
# }
# stdin {
# codec => multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
# }
}
#filter {
#multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
#}
output {
stdout {
codec => rubydebug
}
# elasticsearch {
# hosts => ["172.25.20.1"]
# index => "apache-%{+YYYY.MM.dd}"
# }
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "172.25.20.250 - - [26/Oct/2018:15:11:00 +0800] \"GET / HTTP/1.1\" 403 3985 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
"@version" => "1",
"@timestamp" => "2018-10-26T07:11:01.598Z",
"path" => "/var/log/httpd/access_log",
"host" => "server1"
}
[root@server1 conf.d]# vim test.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf
Settings: Default pipeline workers: 1
Pipeline main started
55.3.244.1 GET /index.html 15824 0.043
{
"message" => "55.3.244.1 GET /index.html 15824 0.043",
"@version" => "1",
"@timestamp" => "2018-10-26T07:16:11.459Z",
"host" => "server1",
"client" => "55.3.244.1",
"method" => "GET",
"request" => "/index.html",
"bytes" => "15824",
"duration" => "0.043"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown
[root@server1 conf.d]# vim /etc/httpd/conf/httpd.conf
[root@server1 conf.d]# vim /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/grok-patterns
[root@server1 conf.d]# vim es.conf
input {
file {
path => "/var/log/httpd/access_log"
start_position => "beginning"
}
# syslog {
# port => 514
# }
# stdin {
# codec => multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
# }
}
#filter {
#multiline {
# pattern => "^\["
# negate => "true"
# what => "previous"
# }
#}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.20.1"]
index => "apache-%{+YYYY.MM.dd}"
}
# file {
# path => "/tmp/testfile"
# codec => line { format => "custom format: %{message}"}
# }
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "172.25.20.250 - - [26/Oct/2018:15:22:35 +0800] \"GET / HTTP/1.1\" 403 3985 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
"@version" => "1",
"@timestamp" => "2018-10-26T07:22:35.549Z",
"path" => "/var/log/httpd/access_log",
"host" => "server1",
"clientip" => "172.25.20.250",
"ident" => "-",
"auth" => "-",
"timestamp" => "26/Oct/2018:15:22:35 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "3985",
"referrer" => "\"-\"",
"agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\""
}
[root@server1 elk]# yum install -y nginx-1.8.0-1.el6.ngx.x86_64.rpm
[root@server1 conf.d]# vim nginx.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.25.20.1"]
index => "nginx-%{+YYYY.MM.dd}"
}
}
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "172.25.20.250 - - [26/Oct/2018:16:04:39 +0800] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\" \"-\"",
"@version" => "1",
"@timestamp" => "2018-10-26T08:04:40.693Z",
"path" => "/var/log/nginx/access.log",
"host" => "server1",
"clientip" => "172.25.20.250",
"ident" => "-",
"auth" => "-",
"timestamp" => "26/Oct/2018:16:04:39 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "200",
"bytes" => "612",
"referrer" => "\"-\"",
"agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
"xforward" => "\"-\""
}
[root@server1 elk]# yum install -y kibana-4.5.1-1.x86_64.rpm
[root@server1 elk]# cd /opt/kibana/config/
[root@server1 config]# vim kibana.yml
elasticsearch.url: "http://172.25.20.1:9200"
kibana.index: ".kibana"
[root@server1 config]# /etc/init.d/kibana start
kibana started
[root@server1 config]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "172.25.20.250 - - [26/Oct/2018:16:15:46 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\" \"-\"",
"@version" => "1",
"@timestamp" => "2018-10-26T08:15:47.465Z",
"path" => "/var/log/nginx/access.log",
"host" => "server1",
"clientip" => "172.25.20.250",
"ident" => "-",
"auth" => "-",
"timestamp" => "26/Oct/2018:16:15:46 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "304",
"bytes" => "0",
"referrer" => "\"-\"",
"agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
"xforward" => "\"-\""
}
Server2:
[root@server2 ~]# yum install -y gcc
[root@server2 ~]# tar zxf redis-4.0.8.tar.gz
[root@server2 ~]# cd redis-4.0.8
[root@server2 redis-4.0.8]# make
[root@server2 redis-4.0.8]# make install
[root@server2 redis-4.0.8]# cd utils/
[root@server2 utils]# ./install_server.sh
[root@server2 utils]# vim /etc/redis/6379.conf
bind 0.0.0.0
[root@server2 utils]# /etc/init.d/redis_6379 restart
Stopping ...
Redis stopped
Starting Redis server...
Server1:
[root@server1 conf.d]# vim nginx.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
}
}
output {
redis {
host => ["172.25.20.2"]
port => 6379
data_type => "list"
key => "logstashtoredis"
}
}
[root@server1 conf.d]# ll /var/log/nginx/access.log
-rw-r----- 1 nginx adm 1045 Oct 26 16:15 /var/log/nginx/access.log
[root@server1 conf.d]# chmod 644 /var/log/nginx/access.log
[root@server1 conf.d]# /etc/init.d/logstash restart
Killing logstash (pid 3313) with SIGTERM
Waiting logstash (pid 3313) to die...
Waiting logstash (pid 3313) to die...
logstash stopped.
logstash started.
Server3:
[root@server3 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm
Preparing... ########################################### [100%]
1:logstash ########################################### [100%]
[root@server3 elk]# cd /etc/logstash/conf.d/
[root@server3 conf.d]# vim redis.conf
input {
redis {
host => ["172.25.20.2"]
port => 6379
data_type => "list"
key => "logstashtoredis"
}
}
output {
elasticsearch {
hosts => ["172.25.20.1"]
index => "nginx-%{+YYYY.MM.dd}"
}
}
[root@server3 conf.d]# /etc/init.d/logstash restart
Killing logstash (pid 1412) with SIGTERM
Waiting logstash (pid 1412) to die...
Waiting logstash (pid 1412) to die...
logstash stopped.
logstash started.
