Logstash - JSON格式文件转换

一份简单可用的JSON转换配置
test.log

{"time":1526659044,"data":"{\"appKey\":\"7b9a2890-5754-11e8-983c-6b4bcc3b7c2e\",\"channelCode\":\"\",\"channelId\":\"\",\"createDateTime\":1526659043119,\"path\":\"\",\"query\":{},\"retain\":\"17670\",\"shareId\":\"\",\"scene\":\"\",\"type\":\"login_out\",\"userId\":\"\"}"}
{"time":1526659044,"data":"{\"appKey\":\"7b9a2890-5754-11e8-983c-6b4bcc3b7c2e\",\"channelCode\":\"\",\"channelId\":\"\",\"createDateTime\":1526659043119,\"path\":\"\",\"query\":{},\"retain\":\"17670\",\"shareId\":\"\",\"scene\":\"\",\"type\":\"login_out\",\"userId\":\"asdfas-asdf_sad\"}"}
{"time":1526659044,"data":"{\"appKey\":\"7b9a2890-5754-11e8-983c-6b4bcc3b7c2e\",\"channelCode\":\"\",\"channelId\":\"\",\"createDateTime\":1526659043119,\"path\":\"\",\"query\":{},\"retain\":\"17670\",\"shareId\":\"\",\"scene\":\"\",\"type\":\"login_out\",\"userId\":\"asdfas-asdf_sad\"}"}
{"time":1527844300209,"data":"{\"appKey\":\"7b9a2890-5754-11e8-983c-6b4bcc3b7c2e\",\"channelCode\":\"\",\"channelId\":\"\",\"createDateTime\":1526659043119,\"path\":\"\",\"query\":{},\"retain\":\"17670\",\"shareId\":\"\",\"scene\":\"\",\"type\":\"login_out\",\"userId\":\"asdfas-asdf_sad\"}"}

json.conf

input { 
    file {
        path => '/home/lake/dounine/github/logstash/test.log'
        start_position => "beginning"
        sincedb_path => "/home/lake/dounine/github/logstash/access_progress.log"
     }
}
filter{
    mutate{
        gsub => [
            "message", '\\"', '"',
            "message", '"{', '{',
            "message", '}"', '}'
        ]
    }
    json{
        source => "message"
        add_field => {
            "uid" => "%{[data][userId]}"
        }
        add_field => {
            "gsm" => "%{[data][channelCode]}"
        }
        add_field => {
            "sid" => "%{[data][shareId]}"
        }
        add_field => {
            "type" => "%{[data][type]}"
        }
        add_field => {
            "ak" => "%{[data][appKey]}"
        }
    }
    if[gsm]!=''{
        mutate{
            split => ["gsm" , "-"]
            add_field =>  [
                "gname","%{[gsm][0]}"
            ]
            add_field =>  [
                "sence","%{[gsm][1]}"
            ]
            add_field =>  [
                "material","%{[gsm][2]}"
            ]
        }
        mutate{
            join => ["gsm" , "_"]
        }
    }else{
        mutate{
            remove_field => ["gsm"]
        }
    }

    if[sid]==''{
        mutate{
            remove_field => ["sid"]
        }
    }
    date {
        match => ["time", "UNIX"]  #UNIX_MS
        target => "time"
        #add_field => {"date"=>"%{+yyyy-M-dd}"}
    }
    ruby {
       code => "event.set('time', event.get('time').time.localtime + 8*60*60)" #+8小时
    }
    mutate{
        gsub => [
            "uid", '-', '_' ,
            "sid", '-', '_' ,
            "ak", '-', '_'
        ]
        remove_field => ["message","host","@version","userId","data","path"]
    }
}

output{
  if[uid]!=''{
    stdout{ codec => rubydebug }
    elasticsearch{
        hosts => ["127.0.0.1:9200"]
        index => "analysis3"
        document_type => "analysis_log"
    }
  }
}

---