ELK（ElasticSearch+Logstash+Kiabana+redis）实时日志分析平台环境部署流程
一、系统环境及软件版本
CentOS release 6.9 (Final)
java 1.8
elasticsearch-2.x
logstash-2.1
kibana-4.3.1-linux-x64
redis 3.2.12
服务器：192.168.1.248
二、安装ElasticSearch
基础环境安装
1）下载并安装GPG Key
rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
2）添加yum仓库
[root@new ~]# vim /etc/yum.repos.d/elasticsearch.repo 【将里面内容情况，配置下面内容】
[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
3）安装elasticsearch
[root@new ~]# yum install -y elasticsearch
配置部署
1）配置修改配置文件
[root@new ~]# mkdir -p /data/es-data
[root@new ~]# vim /etc/elasticsearch/elasticsearch.yml 【将里面内容情况，配置下面内容】
cluster.name: huanqiu # 组名（同一个组，组名必须一致）
node.name: elk-node1 # 节点名称，建议和主机名一致
path.data: /data/es-data # 数据存放的路径
path.logs: /var/log/elasticsearch/ # 日志存放的路径
bootstrap.mlockall: true # 锁住内存，不被使用到交换分区去
network.host: 192.168.1.248 # 网络设置
http.port: 9200 # 端口
2）启动并查看
[root@new ~]# chown -R elasticsearch.elasticsearch /data/
[root@new ~]# service elasticsearch start
[root@new ~]# service elasticsearch status
CGroup: /system.slice/elasticsearch.service
└─3005 /bin/java -Xms256m -Xmx1g -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSI...
[root@new src]# netstat -antlp |egrep "9200|9300"
tcp6 0 0 :::9200 :::* LISTEN 3005/java
tcp6 0 0 :::9300 :::* LISTEN 3005/java
3）接下来安装head插件，使用插件进行查看
首先下载head插件，下载到/usr/loca/src目录下
下载地址：https://github.com/mobz/elasticsearch-head
[root@new src]# unzip elasticsearch-head-master.zip
[root@new src]# ls
elasticsearch-head-master elasticsearch-head-master.zip
[root@new src]# cd /usr/share/elasticsearch/plugins/
[root@new plugins]# mkdir head
[root@new plugins]# ls
head
[root@new plugins]# cd head
[root@new head]# cp -r /usr/local/src/elasticsearch-head-master/* ./
[root@new head]# pwd
/usr/share/elasticsearch/plugins/head
[root@new head]# chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins
[root@new head]# ll
total 40
[root@new _site]# service elasticsearch start
插件访问地址 http://192.168.1.248:9200/_plugin/head/
三 Logstash安装配置
1）下载并安装GPG Key
[root@new ~]# rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
2）添加yum仓库
[root@new ~]# vim /etc/yum.repos.d/logstash.repo
[logstash-2.1]
name=Logstash repository for 2.1.x packages
baseurl=http://packages.elastic.co/logstash/2.1/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
3）安装logstash
[root@new ~]# yum install -y logstash
4）logstash的配置和文件的编写
[root@new ~]# vim shipper.conf
input {
file {
path => "/var/log/elasticsearch/huanqiu.log"
type => "es-error"
start_position => "beginning"
}
}
output {
if [type] == "es-error"{
elasticsearch {
hosts => ["192.168.1.248:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
}
5）执行如下命令收集：
[root@new ~]# /opt/logstash/bin/logstash -f shipper.conf
6）登陆elasticsearch界面，查看数据：
7）ES基本查询及HEAD操作选项解析参考网站地址
https://blog.csdn.net/u013400939/article/details/81008581
四 Kibana安装配置
1）kibana的安装：
[root@new ~]# cd /usr/local/src
[root@new src]# wget https://download.elastic.co/kibana/kibana/kibana-4.3.1-linux-x64.tar.gz
[root@new src]# tar zxf kibana-4.3.1-linux-x64.tar.gz
[root@new src]# mv kibana-4.3.1-linux-x64 /usr/local/
[root@new src]# ln -s /usr/local/kibana-4.3.1-linux-x64/ /usr/local/kibana
2）修改配置文件：
[root@new config]# pwd
/usr/local/kibana/config
[root@new config]# cp kibana.yml kibana.yml.bak
[root@new config]# vim kibana.yml
server.port: 5601
server.host: "192.168.1.248"
elasticsearch.url: "http://192.168.1.248:9200"
kibana.index: ".kibana"
[root@new ~]# yum -y install screen
[root@new ~]# screen ＃这样就另开启了一个终端窗口
[root@new ~]# /usr/local/kibana/bin/kibana
log [18:23:19.867] [info][status][plugin:kibana] Status changed from uninitialized to green - Ready
log [18:23:19.911] [info][status][plugin:elasticsearch] Status changed from uninitialized to yellow - Waiting for Elasticsearch
log [18:23:19.941] [info][status][plugin:kbn_vislib_vis_types] Status changed from uninitialized to green - Ready
log [18:23:19.953] [info][status][plugin:markdown_vis] Status changed from uninitialized to green - Ready
log [18:23:19.963] [info][status][plugin:metric_vis] Status changed from uninitialized to green - Ready
log [18:23:19.995] [info][status][plugin:spyModes] Status changed from uninitialized to green - Ready
log [18:23:20.004] [info][status][plugin:statusPage] Status changed from uninitialized to green - Ready
log [18:23:20.010] [info][status][plugin:table_vis] Status changed from uninitialized to green - Ready
screen相关操作命令解析地址 https://justdo2008.iteye.com/blog/1888772
3）访问kibana：http://192.168.1.248:5601/
五 redis的安装配置
1）redis的安装：
#提前先下载安装epel源：epel-release-latest-7.noarch.rpm，否则yum会报错:No Package.....
[root@new ~]# wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
[root@new ~]# rpm -ivh epel-release-latest-7.noarch.rpm
#安装Redis
[root@new ~]# yum install -y redis
2）redis的配置和启动
[root@new ~]# vim /etc/redis.conf #修改下面两行内容
daemonize yes
bind 192.168.1.248
[root@new ~]# service redis start
[root@new ~]# lsof -i:6379
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
redis-ser 19474 redis 4u IPv4 1344465 0t0 TCP elk-node1:6379 (LISTEN)
[root@new ~]# redis-cli -h 192.168.1.248
192.168.1.248:6379> info
# Server
redis_version:2.8.19
.......
3） logstash文件配置
[root@new ~]# vim shipper.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/huanqiu.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
}
output {
if [type] == "system"{
redis {
host => "192.168.1.248"
port => "6379"
db => "6"
data_type => "list"
key => "system"
}
}
if [type] == "es-error"{
redis {
host => "192.168.1.248"
port => "6379"
db => "6"
data_type => "list"
key => "demo"
}
}
}
[root@new ~]# /opt/logstash/bin/logstash -f shipper.conf
4） elasticSearch文件配置
[root@new ~]# cat file.conf
input {
redis {
type => "system"
host => "192.168.1.248"
port => "6379"
db => "6"
data_type => "list"
key => "system"
}
redis {
type => "es-error"
host => "192.168.1.248"
port => "6379"
db => "6"
data_type => "list"
key => "es-error"
}
}
output {
if [type] == "system"{
elasticsearch {
hosts => ["192.168.1.248:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
if [type] == "es-error"{
elasticsearch {
hosts => ["192.168.1.248:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
}
[root@new ~]# /opt/logstash/bin/logstash -f file.conf
六 详细流程参考地址集合
主流程地址 https://www.cnblogs.com/kevingrace/p/5919021.html
kibana搜索框无法修改bug解决方案的地址 https://blog.csdn.net/u010454030/article/details/70327723