day22(ELK第三天）

1、回顾昨天(索引优化）

1. 按照我们想要的效果生成索引
- 自定索引名称
- 按月生成
- 按域名和访问类型创建索引
- 能够索引日志里的 每个字内容

2. filebeat设置
- nginx格式修改为json
- filebeate添加模板相关选项
       setup.template.name: "nginx"
       setup.template.pattern: "nginx_*"
       setup.template.enabled: false
       setup.template.overwrite: true
- filebeat添加参数直接解析成json格式
      json.keys_under_root: true
      json.overwrite_keys: true
- filebeat的input添加tags标签
      tags["www"]
- filwbeat的output里判断tags
    - index: "nginx_www_access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "www"


3. 收集tomcat日志
- 安装tomcat并访问，产生数据
- 修改tomcat日志为json格式，139行替换
- 验证tomcat日志是否为json格式
- filebeat配置
-- 使用*来匹配tomcat每天生成的日志
    - type: log


4. 收集java多行日志
- filebeat配置java日志路径
- 添加3行多行匹配的参数
  - type: log
    enabled: true 
    paths:
      - /var/log/elasticsearch/linux58.log
    tags: ["java"]
    multiline.pattern: '^\['
    multiline.negate: true
    multiline.match: after


5.kibana画图展示
- 柱状图，饼图，仪表图，折线图，Data Table,markdown
- Dashboard大屏实时展示
- 查询时间要注意，查询条件对面板也是生效的
- 画完记得保存
- 别把es里的.kibana给删了

2、ELk filebeat modules

filebeat是go语言编写
- 快，不依赖于java环境

配置步骤：
1.配置相关modules参数
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s

2.查看模块
filebeat modules list

3.激活模块
[root@db01 ~]# filebeat modules enable nginx
Enabled nginx

4.修改nginx为普通日志格式

5.修改filebeat配置文件


不足的地方：
1.错误日志和正确日志都混在一起了
2.不能按域名生成索引

2.1、filebeat modules 自定义索引和视图

1.为了不影响实验，建议删除所有其他的索引
systemctl stop elasticsearch
systemctl stop kibana
rm -rf /data/elasticsearch/*
rm -rf /var/lib/kibana/*
systemctl start elasticsearch
systemctl start kibana

2.修改nginx配置文件
sed -i 's#json#main#g' /etc/nginx/conf.d/bbs.conf

3.清空nginx日志
> /var/log/nginx/bbs_access.log

4.重启nginx
systemctl restart nginx

5.修改filebeat配置文件:
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
setup.kibana:
  host: "10.0.0.51:5601"
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  indices:
    - index: "nginx_bbs_access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        source: "/var/log/nginx/bbs_access.log"
    - index: "nginx_error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        fileset.name: "error"
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true

6.激活nginx模块报错
filebeat modules enable nginx

7.安装nginx modules插件
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-geoip-6.6.0.zip /usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-user-agent-6.6.0.zip 

8.重启es
systemctl restart elasticsearch

9.修改模块配置
[root@db01 ~]# egrep -v "#|^$" /etc/filebeat/modules.d/nginx.yml 
- module: nginx
  access:
    enabled: true
    var.paths: ["/var/log/nginx/bbs_access.log"]
  error:
    enabled: true
    var.paths: ["/var/log/nginx/error.log"]


10.备份删除不必要的视图文件并导入到kibana
cp -a /usr/share/filebeat/kibana /root
cd /usr/share/filebeat/kibana/6/dashboard
find . -type f ! -name "*nginx*"|xargs rm -rf
rm -rf  ml-nginx-*
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-logs.json 
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-overview.json
cd index-pattern/
sed -i 's#filebeat\-\*#nginx\_\*#g' filebeat.json
filebeat setup --dashboards -E setup.dashboards.directory=/root/kibana/
rm -rf /var/lib/kibana/*
systemctl restart kibana

3、安装docker步骤

安装docker步骤
rm -fr /etc/yum.repos.d/local.repo
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.ustc.edu.cn/docker-ce/linux/centos/docker-ce.repo
sed -i 's#download.docker.com#mirrors.tuna.tsinghua.edu.cn/docker-ce#g' /etc/yum.repos.d/docker-ce.repo
yum install docker-ce -y
systemctl start docker
cat > /etc/docker/daemon.json <

3.1、收集docker容器日志

1.生成多个容器
systemctl stop nginx 
docker stop $(docker ps -q)
docker rm $(docker ps -aq)
docker commit nginx nginx:v2
docker run --name nginx -p 80:80 -d nginx
docker run --name mysql -p 8080:80 -d nginx:v2
docker images
docker ps 
docker logs -f nginx
docker logs -f mysql


2.修改filebeat配置文件
filebeat.inputs:
- type: docker
  containers.ids: 
    - '*'
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false 
setup.kibana:
  host: "10.0.0.51:5601"
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  index: "docker-nginx-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true

3.重启filebeat
systemctl restart filebeat

3.2、按服务类型拆分docker容器日志

1.安装docker-compose
yum install -y python2-pip
2.这里使用pip安装，默认源为国外，可以使用国内加速，相关网站
https://mirrors.tuna.tsinghua.edu.cn/help/pypi/
pip加速操作命令
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pip -U
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
3.继续安装docker-compose
pip install docker-compose
4.检查版本
docker-compose version

5.创建docker-compose配置文件
[root@db03 ~]# cat docker-compose.yml 
version: '3'
services:
  nginx:
    image: nginx:latest
    #设置labels
    labels:
      service: nginx
    #logging设置增加labels.service
    logging:
      options:
        labels: "service"
    ports:
      - "80:80"
  mysql:
    image: nginx:v2
    #设置labels
    labels:
      service: mysql
    #logging设置增加labels.service
    logging:
      options:
        labels: "service"
    ports:
      - "8080:80"


6.使用docker-compose启动docker
docker stop $(docker ps -q)
docker rm $(docker ps -aq)
docker-compose up -d
docker ps 

7.修改filebeat配置文件
filebeat.inputs:
- type: log 
  enabled: true 
  paths:
    - /var/lib/docker/containers/*/*-json.log
  json.keys_under_root: true
  json.overwrite_keys: true

setup.kibana:
  host: "10.0.0.51:5601"

output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  indices:
    - index: "docker-nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "nginx"
        stream: "stdout"
    - index: "docker-nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "nginx"
        stream: "stderr"
    - index: "docker-mysql-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "mysql"
        stream: "stdout"
    - index: "docker-mysql-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "mysql"
        stream: "stderr"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true    

4、filebeat和logstash配置

filebeat存入redis，logstash从redis读取数据

#filebeat配置
filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/bbs_access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["bbs"]

- type: log
  enabled: true 
  paths:
    - /var/log/nginx/www_access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["www"]

setup.kibana:
  host: "10.0.0.51:5601"

output.redis:
  hosts: ["localhost"]
  keys:
    - key: "bbs"
      when.contains:
        tags: "bbs"
    - key: "www"
      when.contains:
        tags: "www"
  db: 0
  timeout: 5

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

#redis查看命令
redis-cli 
keys *
llen bbs
llen www


#logstash配置
[root@db01 /data/soft]# cat /etc/logstash/conf.d/redis.conf 
input {
  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "bbs"
    data_type => "list"
  }

  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "www"
    data_type => "list"
  }
}



#filter {
#  mutate {
#    convert => ["upstream_time", "float"]
#    convert => ["request_time", "float"]
#  }
#}

output {
    if "bbs" in [tags] {
      stdout {} 
      elasticsearch {
        hosts => "http://10.0.0.51:9200"
        manage_template => false
        index => "nginx-bbs-%{+yyyy.MM}"
      }
    }
   
    if "www" in [tags] {
      stdout {} 
      elasticsearch {
        hosts => "http://10.0.0.51:9200"
        manage_template => false
        index => "nginx-www-%{+yyyy.MM}"
      }
    }
}

#logstash启动命令
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis.conf

4.1、redis和logstash配置优化

#####filebeat#######
filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/bbs_access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["bbs"]

- type: log
  enabled: true 
  paths:
    - /var/log/nginx/www_access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["www"]

setup.kibana:
  host: "10.0.0.51:5601"

output.redis:
  hosts: ["localhost"]
  key: "all_keys"
  db: 0
  timeout: 5

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

#######logstash########
input {
  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "all_keys"
    data_type => "list"
  }

#filter {
#  mutate {
#    convert => ["upstream_time", "float"]
#    convert => ["request_time", "float"]
#  }
#}

output {
    if "bbs" in [tags] {
      stdout {} 
      elasticsearch {
        hosts => "http://10.0.0.51:9200"
        manage_template => false
        index => "nginx-bbs-%{+yyyy.MM}"
      }
    }
   
    if "www" in [tags] {
      stdout {} 
      elasticsearch {
        hosts => "http://10.0.0.51:9200"
        manage_template => false
        index => "nginx-www-%{+yyyy.MM}"
      }
    }
}