Filebeat Logstash crontab 定时采集日志
一、系统环境及软件版本
CentOS release 6.9 (Final)
java 1.8
logstash-6.6
filebeat 6.6
crontab
二、Filebeat部署
1）下载filebeat源码包
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.6.0-linux-x86_64.tar.gz
2）解压源码包
tar xvf filebeat-6.6.0-linux-x86_64.tar.gz -C /usr/local/
mv /usr/local/filebeat-6.6.0-linux-x86_64/ /usr/local/filebeat
3）修改配置文件
vim /usr/local/filebeat/filebeat.yml
filebeat.inputs:
- type: log
# Change to true to enable this input configuration.
enabled: true
# 可以设置多个被抓取的目录
# 指定读取文件的位置
paths:
- /var/log/*.log
- /data/logs/rpc/shop-rpc/*.log
output.logstash:
# The Logstash hosts
hosts: ["10.3.2.4:12432"]
4）配置环境变量
vim /etc/profile.d/elk.sh
export PATH=$PATH:/usr/local/filebeat/
source /etc/profile
5）创建并编写脚本
mkdir /usr/local/sh
cd /usr/local/sh
vim filebeat.sh 编写filebeat启动脚本
#!/bin/bash
#添加环境变量
source /etc/profile
nohup filebeat -e -c /usr/local/filebeat/filebeat.yml > /dev/null 2> /dev/null &chmod 777 filebeat.sh 赋予读写运行的权限
vim stop.sh 编写filebeat关闭脚本
#!/bin/bash
pid=$( ps -ef|grep filebeat|grep -v grep|awk '{print $2}' )
if [ -n "$pid" ]
then
kill -9 $pid
fi
chmod 777 stop.sh 赋予读写运行的权限
6）安装和配置crontab定时任务
service crond status 判断是否已安装crontab
安装流程
yum -y install vixie-cron
yum -y install crontabs
配置任务
crontab -e
* 0 * * * /usr/local/sh/filebeat.sh #零点开启filebeat抓取日志
* 1 * * * /usr/local/sh/stop.sh #凌晨一点关闭filebeat
/sbin/service crond reload #重新载入配置
/sbin/service crond restart #重启服务
三、logstash部署
1）下载Logstash源码包
wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.0.tar.gz
2）解压源码包
tar xvf logstash-6.6.0.tar.gz -C /usr/local/
mv /usr/local/logstash-6.6.0/ /usr/local/logstash
3）填写配置文件
vim /usr/local/logstash/config/test.conf
input {
beats {
port => 12432
}
}
filter {
mutate {
#数字2可设置为想过滤的文件夹层级，正则替换
gsub => ["source" , "^(\/[^\/^]+){0}/","" ]
add_field => { "remote_ip" => "%{[@metadata][ip_address]}" }
}
}
output {
file {
path => "/var/backup/%{remote_ip}/%{source}"
codec => line { format => "custom format: %{message}"}
}
}
4）创建并编写脚本
mkdir /usr/local/sh
cd /usr/local/sh
vim logstash.sh 编写logstash启动脚本
#!/bin/bash
#添加环境变量
source /etc/profile
nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/config/test.conf > /dev/null 2> /dev/null &
chmod 777 logstash.sh 赋予读写运行的权限
vim stopLogstash.sh 编写logstash关闭脚本
#!/bin/bash
pid=$( ps -ef|grep logstash|grep -v grep|awk '{print $2}' )
if [ -n "$pid" ]
then
kill -9 $pid
fi
chmod 777 stopLogstash.sh 赋予读写运行的权限
5）安装和配置crontab定时任务
service crond status 判断是否已安装crontab
安装流程
yum -y install vixie-cron
yum -y install crontabs
编辑任务
crontab -e
50 23 * * * /usr/local/sh/logstash.sh #下午十一点五十启动logstash接受日志
* 1 * * * /usr/local/sh/stopLogstash.sh #凌晨一点关闭logstash
/sbin/service crond reload #重新载入配置
/sbin/service crond restart #重启服务
四 详细流程参考地址集合
主流程地址 https://www.cnblogs.com/kevingrace/p/5919021.html
https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/output/file.html
https://blog.csdn.net/yelllowcong/article/details/80847718
centos crontab用法详解 https://blog.csdn.net/zyddj123/article/details/83109716