安装JAVA环境
[root@tzgsqvelk01 tzg]# yum install java-1.8.0-openjdk
[root@tzgsqvelk01 tzg]# export JAVA_HOME=/usr/java
Elastic Stack安装步骤
- 1、Elasticsearch
X-Pack for Elasticsearch - 2、Kibana
X-Pack for Kibana - 3、Logstash
- 4、Beats
- 5、Elasticsearch Hadoop
- 6、redis
参考文档
https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html
1、安装Elasticsearch & X-pack（yum安装）
X-Pack是一个Elastic Stack的扩展，将安全，警报，监视，报告和图形功能包含在一个易于安装的软件包中。x-Pack安装之后有一个超级用户elastic ，其默认的密码是changeme，拥有对所有索引和数据的控制权，可以使用该用户创建和修改其他用户。通过kibana的web界面进行用户和用户组的管理，也可以使用shell 终端进行管理。
[root@tzgsqvelk01 tzg]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/elasticsearch.repo <<-EOF
[elasticsearch-5.x]
name=Elasticsearch repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
[root@tzgsqvelk01 tzg]# yum install elasticsearch -y
[root@tzgsqvelk01 tzg]# /bin/systemctl daemon-reload
[root@tzgsqvelk01 tzg]# /bin/systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
[root@tzgsqvelk01 tzg]# grep -v "^#" /etc/elasticsearch/elasticsearch.yml
node.name: tzg-elk01
path.data: /tzgData/elk/elasticsearch/data
path.logs: /opt/elk/elasticsearch/logs
network.host: 0.0.0.0
[root@tzgsqvelk01 tzg]# mkdir -p /tzgData/elk/elasticsearch/data
[root@tzgsqvelk01 tzg]# mkdir -p /opt/elk/elasticsearch/logs
[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /tzgData/elk/
[root@tzgsqvelk01 tzg]# chown -R elasticsearch.elasticsearch /opt/elk/
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# curl http://localhost:9200
{
"name" : "tzg-elk01",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "K8bEym6OSpm9o1Y4CJrrEg",
"version" : {
"number" : "5.1.1",
"build_hash" : "5395e21",
"build_date" : "2016-12-06T12:36:15.409Z",
"build_snapshot" : false,
"lucene_version" : "6.3.0"
},
"tagline" : "You Know, for Search"
}
[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/elastic/_password' -d '{
"password" : "123456"
}'
Enter host password for user 'elastic':
{}[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# systemctl stop elasticsearch.service
[root@tzgsqvelk01 tzg]# /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack
[root@tzgsqvelk01 tzg]# systemctl start elasticsearch.service
kibana & x-pack安装
[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/kibana.repo <<-EOF
[kibana-5.x]
name=Kibana repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]#
[root@tzgsqvelk01 tzg]# yum install kibana -y
[root@tzgsqvelk01 tzg]# grep "^[a-z]" /etc/kibana/kibana.yml
server.host: "172.16.5.37"
elasticsearch.username: "elastic"
elasticsearch.password: "123456"
[root@tzgsqvelk01 tzg]# systemctl stop kibana.service
[root@tzgsqvelk01 tzg]# /usr/share/kibana/bin/kibana-plugin install file:///home/x-pack-5.1.1.zip
[root@tzgsqvelk01 tzg]# systemctl start kibana.service
[root@tzgsqvelk01 tzg]# curl -XPUT -u elastic 'localhost:9200/_xpack/security/user/kibana/_password' -d '{
"password" : "123456"
}'
Enter host password for user 'elastic':
logstash 安装
[root@tzgsqvelk01 tzg]# cat > /etc/yum.repos.d/logstash.repo <<-EOF
[logstash-5.x]
name=Elastic repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
[root@tzgsqvelk01 tzg]# yum install logstash -y
[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "172.16.5.37:9200" user => elastic password => 123456 } } '
hahahha
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
The stdin plugin is now waiting for input:
10:47:39.397 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://~hidden~:[email protected]:9200"]}}
10:47:39.400 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:url=>#, :healthcheck_path=>"/"}
10:47:39.617 [[main]-pipeline-manager] WARN logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#}
10:47:39.617 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}
10:47:39.658 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
10:47:39.772 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Installing elasticsearch template to _template/logstash
10:47:39.815 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["172.16.5.37:9200"]}
10:47:39.817 [[main]-pipeline-manager] INFO logstash.pipeline - Starting pipeline {"id"=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>1000}
10:47:39.823 [[main]-pipeline-manager] INFO logstash.pipeline - Pipeline main started
10:47:39.856 [Api Webserver] INFO logstash.agent - Successfully started Logstash API endpoint {:port=>9600}
^C10:47:47.587 [SIGINT handler] WARN logstash.runner - SIGINT received. Shutting down the agent.
10:47:47.592 [LogStash::Runner] WARN logstash.agent - stopping pipeline {:id=>"main"}
[root@tzgsqvelk01 tzg]# /usr/share/logstash/bin/system-install
Successfully created system startup script for Logstash
[root@tzgsqvelk01 tzg]# chown -R logstash.logstash /usr/share/logstash/data/
redis安装
[root@ tzgsqvelk01 app]# wget http://download.redis.io/releases/redis-3.2.1.tar.gz
[root@ tzgsqvelk01 app]# yum install gcc -y
[root@ tzgsqvelk01 app]# tar zxvf redis-3.2.1.tar.gz
[root@ tzgsqvelk01 app]# cd redis-3.2.1
[root@tzgsqvelk01 redis-3.2.1]# make
[root@tzgsqvelk01 redis-3.2.1]# make install PREFIX=/opt/app/redis
[root@tzgsqvelk01 redis-3.2.1]# cat /opt/app/redis/conf/redis.conf
daemonize yes
pidfile /var/run/redis.pid
bind 172.16.5.37
port 6379
timeout 120
loglevel notice
logfile /opt/app/redis/logs/redis.log
databases 16
save 900 1
save 300 100
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /tzgData/redis/var
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
slowlog-log-slower-than 10000
slowlog-max-len 128
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-entries 512
list-max-ziplist-value 64
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
maxmemory 4096000000
maxclients 10000
slave-serve-stale-data yes
slave-read-only yes
slave-priority 100
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
[root@tzgsqvelk01 redis-3.2.1]# cp utils/redis_init_script /etc/init.d/redisd
[root@tzgsqvelk01 redis-3.2.1]# vim /etc/init.d/redisd
#!/bin/sh
# chkconfig: 2345 90 10
# description: Redis is a persistent key-value database
#
REDISPORT=6379
EXEC=/opt/app/redis/bin/redis-server
CLIEXEC=/opt/app/redis/bin/redis-cli
PIDFILE=/var/run/redis_${REDISPORT}.pid
CONF="/opt/app/redis/conf/redis.conf"
[root@tzgsqvelk01 tzg]# chkconfig redisd on
[root@tzgsqvelk01 tzg]# service redisd start
[root@tzgsqvelk01 app]# rm -rf redis-3.2.1
[root@tzgsqvelk01 app]# rm -rf redis-3.2.1.tar.gz
logstash和elasticsearch 无需授权配置
参考官方文档
https://www.elastic.co/guide/en/x-pack/current/anonymous-access.html
[root@ tzgsqvelk01 elasticsearch]# vim elasticsearch.yml
xpack.security.authc:
anonymous:
username: anonymous_user
roles: logstash_logs
authz_exception: true
######添加 logstash_logs的权限######
curl -XPOST -u elastic 'localhost:9200/_xpack/security/role/logstash_logs' -d '{
"cluster": ["manage_index_templates", "monitor"],
"indices": [
{
"names": [ "logstash-*","business-index-*"],
"privileges": ["write","delete","create_index"]
}
]
}'
[root@ tzgsqvelk01 elasticsearch]# systemctl restart elasticsearch.service
[root@ tzgsqvelk01 elasticsearch]# /usr/share/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch { hosts => "192.168.3.210:9200"} } '
tzghahaha
Sending Logstash's logs to /var/log/logstash/logs which is now configured via log4j2.properties
The stdin plugin is now waiting for input:
[2016-12-30T08:48:07,332][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>["http://192.168.3.210:9200"]}}
[2016-12-30T08:48:07,342][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:url=>#, :healthcheck_path=>"/"}
[2016-12-30T08:48:07,592][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#}
[2016-12-30T08:48:07,599][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2016-12-30T08:48:07,753][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2016-12-30T08:48:07,773][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["192.168.3.210:9200"]}
[2016-12-30T08:48:07,784][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
[2016-12-30T08:48:07,814][INFO ][logstash.pipeline ] Pipeline main started
[2016-12-30T08:48:07,989][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}