Logstash下载与安装

https://www.elastic.co/cn/products/logstash

下载地址：https://www.elastic.co/cn/downloads/logstash

MacOS

二进制文件下载：点此下载
⚠️注意！请下载与ElasticSearch相同的版本

下载最 MovieLens 最小测试数据集：传送门

编写logstash配置文件：

input {
  file {
    path => "/Users/Angus/Downloads/logstash-7.1.0/bin/movies.csv"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}
filter {
  csv {
    separator => ","
    columns => ["id","content","genre"]
  }

  mutate {
    split => { "genre" => "|" }
    remove_field => ["path", "host","@timestamp","message"]
  }

  mutate {

    split => ["content", "("]
    add_field => { "title" => "%{[content][0]}"}
    add_field => { "year" => "%{[content][1]}"}
  }

#  mutate {

#    gsub => [
#
#      "year", "\\)", ""
#    ]
#  }


  mutate {
    convert => {
      "year" => "integer"
    }
    strip => ["title"]
    remove_field => ["path", "host","@timestamp","message","content"]
  }



}
output {
   elasticsearch {
     hosts => "http://localhost:9200"
     index => "movies"
     document_id => "%{id}"
   }
  stdout {}
}

下载数据集

⚠️注意：将movies.cvs放入文件path => "/Users/Angus/Downloads/logstash-7.1.0/bin/movies.csv"指定的文件夹下！

我的存放目录

> pwd
> /Users/Angus/Downloads/logstash-7.1.0/bin
> ll
-rwxr-xr-x@ 1 Angus  staff   152B  5 16 01:49 benchmark.sh
-rwxr-xr-x@ 1 Angus  staff   377B  5 16 01:49 cpdump
-rwxr-xr-x@ 1 Angus  staff   1.0K  5 16 01:49 dependencies-report
-rwxr-xr-x@ 1 Angus  staff   155B  5 16 01:49 ingest-convert.sh
-rwxr-xr-x@ 1 Angus  staff   2.3K  5 16 01:49 logstash
-rwxr-xr-x@ 1 Angus  staff   447B  5 16 01:49 logstash-keystore
-rw-r--r--@ 1 Angus  staff   257B  5 16 01:49 logstash-keystore.bat
-rwxr-xr-x@ 1 Angus  staff   448B  5 16 01:49 logstash-plugin
-rw-r--r--@ 1 Angus  staff   260B  5 16 01:49 logstash-plugin.bat
-rw-r--r--@ 1 Angus  staff   2.4K  5 16 01:49 logstash.bat
-rw-r--r--  1 Angus  staff   864B  8 22 16:34 logstash.conf
-rwxr-xr-x@ 1 Angus  staff   4.8K  5 16 01:49 logstash.lib.sh
-rw-r--r--  1 Angus  staff   483K  8 22 16:54 movies.csv
-rwxr-xr-x@ 1 Angus  staff   1.1K  5 16 01:49 pqcheck
-rwxr-xr-x@ 1 Angus  staff   1.1K  5 16 01:49 pqrepair
-rwxr-xr-x@ 1 Angus  staff   666B  5 16 01:49 ruby
-rw-r--r--@ 1 Angus  staff   1.4K  5 16 01:49 setup.bat
-rwxr-xr-x@ 1 Angus  staff   3.5K  5 16 01:49 system-install

执行配置文件

> ./logstash -f logstash.conf