22-win10下ElasticSearch.6.1.0与logstash6.1.0的整合

第一次使用logstash，想把它整合到es中

1.logstash安装

下载安装

直接下载：https://www.elastic.co/cn/downloads/logstash，下载6.1.0的版本
解压到D:\elasticsearch\logstash-6.1.0目录下

2 es与logstash整合

2.1.配置

wjb.conf配置文件如下

input { 
    file { 
        path => "D:/elasticsearch/logstash-6.1.0/shakespeare.json"  # 注释，你需要去掉，数据源
        start_position => "beginning" 
        ignore_older => 0 
        codec => "json" 
        # sincedb_path => "/dev/null" 
        type => "wjb_log" 
    } 
} 


filter{ 
    if [bboy_id] { 
        mutate {
            add_tag => "BBOY" 
        } 
    } 
} 


output { 
    if [bboy_id] {
        elasticsearch {
            hosts => "localhost:9200"       # es服务器
            index => "bboy-%{+YYYY.MM.dd}"  
            document_type => "bboy_log" 
        } 
    } else {
        elasticsearch { 
            hosts => "localhost:9200"     # es服务器
            index => "wjb-%{+YYYY.MM.dd}" # es服务器索引格式
            document_type => "wjb_log" 
        } 
    }
}

2.2.数据源

shakespeare.json文件内容如下

{
  "error": false, 
  "results": [
    {
      "_id": "5962c411421aa90ca209c425", 
      "createdAt": "2017-07-10T08:02:25.353Z", 
      "desc": "7-10", 
      "publishedAt": "2017-07-10T12:48:49.297Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fhegpeu0h5j20u011iae5.jpg", 
      "used": true, 
      "who": "daimajia"
    }, 
    {
      "_id": "595ed766421aa90ca209c407", 
      "createdAt": "2017-07-07T08:35:50.172Z", 
      "desc": "7-7", 
      "publishedAt": "2017-07-07T12:14:57.685Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fhb0t7ob2mj20u011itd9.jpg", 
      "used": true, 
      "who": "daimajia"
    }, 
    {
      "_id": "595d82f6421aa90ca3bb6aaa", 
      "createdAt": "2017-07-06T08:23:18.945Z", 
      "desc": "07-06", 
      "publishedAt": "2017-07-06T11:57:03.770Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034gy1fh9utulf4kj20u011itbo.jpg", 
      "used": true, 
      "who": "\u4ee3\u7801\u5bb6"
    }, 
    {
      "_id": "595c2f23421aa90ca209c3f0", 
      "createdAt": "2017-07-05T08:13:23.237Z", 
      "desc": "2017-07-5", 
      "publishedAt": "2017-07-05T11:15:30.556Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fh8ox6bmjlj20u00u0mz7.jpg", 
      "used": true, 
      "who": "daimajia"
    }, 
    {
      "_id": "595ad246421aa90ca3bb6a91", 
      "createdAt": "2017-07-04T07:24:54.820Z", 
      "desc": "7-4", 
      "publishedAt": "2017-07-04T11:50:36.484Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fh7hwi9lhzj20u011hqa9.jpg", 
      "used": true, 
      "who": "daimajia"
    }, 
    {
      "_id": "5941db7b421aa92c794633cd", 
      "createdAt": "2017-06-15T08:57:31.47Z", 
      "desc": "6-15", 
      "publishedAt": "2017-06-15T13:55:57.947Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fgllsthvu1j20u011in1p.jpg", 
      "used": true, 
      "who": "\u4ee3\u7801\u5bb6"
    }, 
    {
      "_id": "593f1ff7421aa92c73b64803", 
      "createdAt": "2017-06-13T07:12:55.795Z", 
      "desc": "6-13", 
      "publishedAt": "2017-06-14T11:34:54.556Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fgj7jho031j20u011itci.jpg", 
      "used": true, 
      "who": "daimajia"
    }, 
    {
      "_id": "593dde44421aa92c73b647f5", 
      "createdAt": "2017-06-12T08:20:20.475Z", 
      "desc": "6-12", 
      "publishedAt": "2017-06-12T11:11:11.25Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fgi3vd6irmj20u011i439.jpg", 
      "used": true, 
      "who": "\u4ee3\u7801\u5bb6"
    }, 
    {
      "_id": "5939fcb1421aa92c7be61bd5", 
      "createdAt": "2017-06-09T09:41:05.305Z", 
      "desc": "6-9", 
      "publishedAt": "2017-06-09T12:50:03.131Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fgepc1lpvfj20u011i0wv.jpg", 
      "used": true, 
      "who": "dmj"
    }, 
    {
      "_id": "5938c377421aa92c7be61bcb", 
      "createdAt": "2017-06-08T11:24:39.838Z", 
      "desc": "6-8", 
      "publishedAt": "2017-06-08T11:27:47.21Z", 
      "source": "chrome", 
      "type": "\u798f\u5229", 
      "url": "https://ws1.sinaimg.cn/large/610dc034ly1fgdmpxi7erj20qy0qyjtr.jpg", 
      "used": true, 
      "who": "daimajia"
    }
  ]
}

2.3 运行es,kibana,head

head

es

kibana

2.4 在es中建立索引，根据当前时间（我现在是2018-03-02）

put /wjb-2018.03.02
{}

如下

2.5运行

命令：

D:\elasticsearch\logstash-6.1.0\bin>logstash.bat -f ../wjb.conf

2.6查看结果

3.配置魔板

整个配置文件分为三部分：input,filter,output
#参考这里的介绍 https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
input {
  #file可以多次使用，也可以只写一个file而设置它的path属性配置多个文件实现多文件监控
  file {
    #type是给结果增加了一个属性叫type值为""的条目。这里的type，对应了ES中index中的type，即如果输入ES时，没有指定type，那么这里的type将作为ES中index的type。
    type => "apache-access" 
    path => "/apphome/ptc/Windchill_10.0/Apache/logs/access_log*"
    #start_position可以设置为beginning或者end，beginning表示从头开始读取文件，end表示读取最新的，这个也要和ignore_older一起使用。
    start_position => beginning
    #sincedb_path表示文件读取进度的记录，每行表示一个文件，每行有两个数字，第一个表示文件的inode，第二个表示文件读取到的位置（byteoffset）。默认为$HOME/.sincedb*
    sincedb_path => "/opt/logstash-2.3.1/sincedb_path/access_progress"
    #ignore_older表示了针对多久的文件进行监控，默认一天，单位为秒，可以自己定制，比如默认只读取一天内被修改的文件。
    ignore_older => 604800
    #add_field增加属性。这里使用了${HOSTNAME}，即本机的环境变量，如果要使用本机的环境变量，那么需要在启动命令上加--alow-env。
    add_field => {"log_hostname"=>"${HOSTNAME}"}
    #这个值默认是\n 换行符，如果设置为空""，那么后果是每个字符代表一个event
    delimiter => ""
    #这个表示关闭超过（默认）3600秒后追踪文件。这个对于multiline来说特别有用。... 这个参数和logstash对文件的读取方式有关，两种方式read tail，如果是read
    close_older => 3600
    coodec => multiline {
      pattern => "^\s"
      #这个negate是否定的意思，意思跟pattern相反，也就是不满足patter的意思。
#      negate => ""
      #what有两个值可选 previous和next，举例说明，java的异常从第二行以空格开始，这里就可以pattern匹配空格开始，what设置为previous意思是空格开头这行跟上一行属于同一event。另一个例子，有时候一条命令太长，当以\结尾时表示这行属于跟下一行属于同一event，这时需要使用negate=>true，what=>'next'。
      what => "previous"
      auto_flush_interval => 60
    }
  }
  file { 
    type => "methodserver-log" 
    path => "/apphome/ptc/Windchill_10.0/Windchill/logs/MethodServer-1604221021-32380.log" 
    start_position => beginning 
    sincedb_path => "/opt/logstash-2.3.1/sincedb_path/methodserver_process"
#    ignore_older => 604800
  }
}
filter{
  #执行ruby程序，下面例子是将日期转化为字符串赋予daytag
  ruby {
    code => "event['daytag'] = event.timestamp.time.localtime.strftime('%Y-%m-%d')"
  }
  # if [path] =~ "access" {} else if [path] =~ "methodserver" {} else if [path] =~ "servermanager" {} else {} 注意语句结构
  if [path] =~ "MethodServer" { #z这里的=~是匹配正则表达式
    grok {
      patterns_dir => ["/opt/logstash-2.3.1/patterns"] #自定义正则匹配
#      Tue 4/12/16 14:24:17: TP-Processor2: hirecode---->77LS
      match => { "message" => "%{DAY:log_weekday} %{DATE_US:log_date} %{TIME:log_time}: %{GREEDYDATA:log_data}"}
    }
    #mutage是做转换用的
    mutate { 
      replace => { "type" => "apache" } #替换属性值
      convert => { #类型转换
        "bytes" => "integer" #例如还有float
        "duration" => "integer"
        "state" => "integer"
      }
    #date主要是用来处理文件内容中的日期的。内容中读取的是字符串，通过date将它转换为@timestamp。参考https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-match
#    date {
#      match => [ "logTime" , "dd/MMM/yyyy:HH:mm:ss Z" ]
#    }
  }else if [type] in ['tbg_qas','mbg_pre'] { # if ... else if ... else if ... else结构
  }else {
    drop{} # 将event丢弃
  }
}
output {
  stdout{ codec=>rubydebug} # 直接输出，调试用起来方便
  # 输出到redis
  redis {
    host => '10.120.20.208'
    data_type => 'list'
    key => '10.99.201.34:access_log_2016-04'
  }
  # 输出到ES
  elasticsearch {
    hosts =>"192.168.0.15:9200"
    index => "%{sysid}_%{type}"
    document_type => "%{daytag}"
  }
}