logstash脚本抽取数据到es或kafka，上

logstash从oracle抽数据到kafka

input {
jdbc {
jdbc_driver_library => “ojdbc6-11.2.0.3.0.jar 插件的”
jdbc_driver_class => “Java::oracle.jdbc.OracleDriver”
jdbc_connection_string => “jdbc:oracle:thin:@//url/库名”
jdbc_user => “用户名”
jdbc_password => “密码”
schedule => “* * * * *”
jdbc_default_timezone => “Etc/UTC”
statement => “select id,birthday,id_type,id_no,last_update_time from sehr_xman “//要抽取的数据
last_run_metadata_path => “/home/elk/logstash-5.4.0/config/meta/ssp_service_journal”
use_column_value => true
tracking_column => “last_update_time”
tracking_column_type => “timestamp”
type => “zoe_agegroup”
add_field => {“device_id” => “zoe-005”}
}
}
output {
kafka{ //抽取到kafka上
topic_id => “zoe_por_agegroup”
bootstrap_servers => “zoe-001:6667,zoe-002:6667”
batch_size => 128
codec => json

    }
    stdout{
        codec => rubydebug
        codec => json
    }
}

命令：启动conf脚本的Linux命令：./logstash -f /home/elk/logstash-5.4.0/zoe_config/test_examine.conf

ps -ef | grep logstash //查看logstash运行的状态

logstash从oracle抽取数据到elasticsearch

input {
jdbc {

            jdbc_driver_library => "ojdbc6-20160518.jar 插件的位置"
            jdbc_driver_class => "Java::oracle.jdbc.OracleDriver"
            jdbc_connection_string => "jdbc:oracle:thin:@//url:1521/库名"
            jdbc_user => "用户名"
            jdbc_password => "密码"
            schedule => "53 9 * * *"
            jdbc_default_timezone => "Etc/UTC"
            statement => "SELECT id,id_no,name,to_char(birth_date,'yyyy-mm-dd') as birthday,sex_code FROM PHIP_XMAN_BASE_INFO" //要抽取到es上的数据
    }

}
filter{
mutate {
remove_field => “@version”
remove_field => “@timestamp”
}
}
output {

    elasticsearch {
            hosts => [ "test-001:9200", "test-002:9200", "test-003:9200", "test-004:9200", "zoe-005:9200"]  //es主机IP
            document_id => "%{id}"
            index => "索引名" //自定义
    }
    #stdout { codec => rubydebug }

}

实时从oracle上面抽取数据到es上面

input {
stdin {
}
jdbc {
# oracle jdbc connection string to our backup databse
jdbc_connection_string => “jdbc:oracle:thin:system/123456@//url:1521/库名”
# the user we wish to excute our statement as
jdbc_user => “system”
jdbc_password => “123456”
# the path to our downloaded jdbc driver
jdbc_driver_library => “/elasticsearch-jdbc-2.3.2.0/lib/ojdbc6.jar”
# the name of the driver class for oracle
jdbc_driver_class => “Java::oracle.jdbc.driver.OracleDriver”

#new add begin 2016-6-28
record_last_run => “true”
use_column_value => “false”
tracking_column => “id”
last_run_metadata_path => “/etc/logstash/run_metadata.d/my_info”
clean_run => “false”
#new add by end

jdbc_paging_enabled => “true”
jdbc_page_size => “50000”
statement_filepath => “/usr/local/logstash/bin/logstash_jdbc_test/jdbc_oracle.sql”
schedule => “* * * * *”
type => “tstype”
}
}

filter {
json {
source => “message”
remove_field => [“message”]
}
#grok {
#match => { “message” => “%{COMBINEDAPACHELOG}” }
#match => { “message” => “test” }

}

date {
match => [ “timestamp” , “dd/MMM/yyyy:HH:mm:ss Z” ]
}
}

output {
elasticsearch {
hosts => “es IP:9200”
index => “tsuser”
document_id => “%{user_id}”
}
stdout {
codec => json_lines
}
}

抽取json数据到es上

input{
file{
start_position => “beginning”
path => [“/root/aaa/aaa/*”] //json文件的位置
sincedb_path=>”/dev/null”
codec => json {
charset => “UTF-8”
}

    }

}
filter{
mutate {
remove_field => “path”
remove_field => “@timestamp”
remove_field => “@version”
remove_field => “host”
}

}

output{
elasticsearch {
hosts => [ “es ip:9200”]
document_id => “%{id}”
index => “ssp-odp-apigateway-2018-08-09-import”
}
}